level=info ts=2026-07-02T02:17:44.570891234Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-07-02T02:17:44.570920989Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-07-02T02:17:44.572075449Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=info ts=2026-07-02T02:17:44.573045123Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-14b3cdd5
level=info ts=2026-07-02T02:17:44.573263364Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-07-02T02:17:44.576686764Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-07-02T02:17:44.578218894Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-07-02T02:17:44.578282089Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
ts=2026-07-02T02:17:44.578469095Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-07-02T02:17:44.578491473Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-07-02T02:17:44.57850229Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=5.252008ms
level=info ts=2026-07-02T02:17:44.578599916Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=info ts=2026-07-02T02:17:44.578646688Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-07-02T02:17:44.578666975Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=error ts=2026-07-02T02:17:44.578723965Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-07-02T02:17:44.578738041Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-07-02T02:17:44.582277243Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-07-02T02:17:44.582291351Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-07-02T02:17:44.596065432Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-07-02T02:17:44.596080834Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-07-02T02:17:44.596124284Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-07-02T02:17:44.596257136Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-07-02T02:17:44.596269828Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-07-02T02:17:44.596294307Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-07-02T02:17:44.596320299Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-07-02T02:17:44.596426796Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-07-02T02:17:44.59645036Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-07-02T02:17:44.596458723Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T02:17:44.596501055Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=21.874µs success=true
level=info ts=2026-07-02T02:17:44.596515248Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-07-02T02:17:44.596617917Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=168.908µs errors=false
level=info ts=2026-07-02T02:17:44.596648368Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-07-02T02:17:44.596783254Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=333.992µs errors=false
level=info ts=2026-07-02T02:17:44.596798781Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-07-02T02:17:44.59680407Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=355.489µs
level=info ts=2026-07-02T02:17:44.596843344Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-07-02T02:17:44.596863406Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-07-02T02:17:44.596846085Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-07-02T02:17:44.597018138Z caller=loki.go:615 msg="Loki started" startup_time=91.167881ms
level=info ts=2026-07-02T02:17:44.596870237Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=3.215821965s
level=info ts=2026-07-02T02:17:44.596950325Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-07-02T02:17:44.597224064Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
ts=2026-07-02T02:17:45.839788851Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-07-02T02:17:45.839818844Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-07-02T02:17:48.242283524Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-07-02T02:17:48.242326065Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-07-02T02:17:49.598328315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:17:54.598252581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-07-02T02:17:55.786609582Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-07-02T02:17:55.786633399Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-07-02T02:17:59.597715594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:18:04.597369983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:18:07.234691799Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=22.65594686s
level=warn ts=2026-07-02T02:18:09.597936369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:18:14.597931822Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:18:14.597958128Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-07-02T02:18:14.597966975Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:18:14.597956244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:18:19.597895122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:18:24.597519835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:18:29.597737663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:18:34.59761788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:18:39.598211668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:18:44.578857901Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:18:44.597301705Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T02:18:44.597299431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:18:44.597329902Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:18:49.598158683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:18:54.597290393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T02:18:58.956867501Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE5786CCA5692B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T02:18:59.597339732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:19:04.597739306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:19:09.597313741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:19:14.597272872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:19:14.597281317Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:19:14.597308754Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:19:19.598098214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:19:24.598109327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:19:29.598110433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:19:34.597585704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:19:39.597788492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:19:41.621160232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:19:41.634696192Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5790BC6D92FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T02:19:44.579316963Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:19:44.597072437Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:19:44.597110422Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:19:44.598217853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:19:49.598181414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:19:54.59782754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:19:59.597468871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:20:00.50961356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:20:00.520421472Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5795221D416F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:20:04.598207221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:20:09.597973095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:20:14.596960352Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:20:14.597001945Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:20:14.598053417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:20:19.597988882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:20:24.598269998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:20:29.597746747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:20:32.332033665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:20:32.342931777Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE579C8AE23F81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:20:34.597893427Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:20:39.597808353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:20:44.578605545Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:20:44.59735046Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:20:44.597382876Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:20:44.597423554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:20:49.598293466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:20:54.597710994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:20:59.598097683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:21:04.598221813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:21:09.597940146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:21:14.597866743Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:21:14.597897997Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:21:14.597893486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:21:19.597296444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:21:21.949115627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:21:21.960527814Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE57A818507120, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:21:24.597953412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:21:29.597999746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:21:34.597672486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:21:39.59732306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:21:44.578605282Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:21:44.59750829Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T02:21:44.59751068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:21:44.597535616Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:21:49.597950129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:21:54.597795939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:21:59.597473606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:22:04.597708986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:22:09.597760053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:22:14.597177156Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:22:14.597210306Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:22:14.598256057Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:22:15.674426964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:22:15.686595968Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE57B49AA3CB67, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:22:19.598197942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:22:24.597248729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:22:29.597984402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T02:22:29.895367683Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE57B7E991D3D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T02:22:34.597310731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:22:39.597765091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:22:44.578697148Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:22:44.596912817Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T02:22:44.597073765Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-07-02T02:22:44.597083951Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T02:22:44.597102817Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.349µs success=true
level=info ts=2026-07-02T02:22:44.597165802Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:22:44.5971932Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:22:44.597340011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:22:49.598139388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:22:54.597763418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:22:59.597461691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:23:03.879388202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:23:03.888903601Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE57BFD3BA1267, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:23:04.597860462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:23:09.597991316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:23:14.597925522Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T02:23:14.59792797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:23:14.597956064Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:23:19.598138452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:23:24.597359726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:23:29.597804069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:23:34.597470545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:23:39.59786821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:23:44.579118069Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:23:44.597202071Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:23:44.597231833Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:23:44.597301867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:23:45.391118254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:23:45.402048189Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE57C97E1A30A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:23:49.59783269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:23:54.598057715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:23:59.597892988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:24:04.597789609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:24:09.597446798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:24:14.596958496Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:24:14.596989376Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:24:14.598107644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:24:19.59749833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:24:24.597274149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:24:29.598144359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:24:34.598070766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:24:39.597546193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:24:43.501119489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:24:43.512748816Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE57D705C38EC3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T02:24:44.578446327Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:24:44.597338418Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:24:44.597375073Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:24:44.597378766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:24:49.597918467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:24:54.597533975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:24:59.598337892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:25:04.597964451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:25:09.597455991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:25:14.597201444Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:25:14.597233529Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:25:14.597284267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:25:19.597438304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:25:24.598182233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:25:29.598299964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:25:32.84607591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:25:32.857239934Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE57E282EB2CC0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:25:34.597617493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:25:39.597448448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:25:44.578949752Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T02:25:44.597847383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:25:44.597865959Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:25:44.597887506Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:25:49.597880126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:25:54.597946442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:25:59.597483423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:26:04.597468454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T02:26:04.92344598Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE57E9FA3D6138, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T02:26:09.597894845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:26:14.597688429Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:26:14.597719618Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:26:14.59773567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:26:19.598012848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:26:20.017820331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:26:20.02957525Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE57ED7EA0F1E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:26:20.029598907Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T02:26:20.029622983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:26:20.038326786Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE57ED7F27E491, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:26:24.597541285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:26:29.597276861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:26:33.662999055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:26:33.674908978Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE57F0ABF23108, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:26:34.597563243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:26:39.598053445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:26:44.578423184Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T02:26:44.597888052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:26:44.597895554Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:26:44.597921021Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:26:49.597478384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:26:54.597269035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:26:59.598281619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:27:04.597433343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:27:07.723605945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:27:07.735571051Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE57F89A1E265C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:27:09.597816865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:27:14.596956963Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:27:14.596991203Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:27:14.598090192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:27:14.603736558Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-07-02T02:27:14.607245696Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009933223s
level=warn ts=2026-07-02T02:27:19.598321481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:27:24.59829533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:27:29.598283964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:27:34.598228575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:27:39.597426125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:27:44.578320431Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:27:44.597211311Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T02:27:44.597268684Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:27:44.59729604Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:27:44.5973039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:27:44.597354436Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-07-02T02:27:44.597361807Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T02:27:44.597379449Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.181µs success=true
level=warn ts=2026-07-02T02:27:49.597924219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:27:54.598048831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:27:59.59787246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:28:04.598305531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:28:06.353517501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:28:06.365380016Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE580640BAFC42, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:28:09.597446445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:28:14.596970338Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:28:14.597002587Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:28:14.598051685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:28:19.59796551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:28:24.597430704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:28:29.597851906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:28:34.597523881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:28:39.597460173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:28:44.578423495Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T02:28:44.597250373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:28:44.597263957Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:28:44.597293171Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:28:49.597850269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:28:54.598266893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:28:59.597452539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:28:59.821631028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:28:59.833110573Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5812B3A6F80C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:29:04.598011836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:29:09.597343078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:29:14.597362887Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T02:29:14.597353641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:29:14.597391445Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:29:19.598220981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:29:24.5972923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:29:29.597970779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:29:34.59753187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:29:39.598280708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:29:44.578869276Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T02:29:44.597907723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:29:44.597929431Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:29:44.597960586Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T02:29:47.772396697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:29:47.782344445Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE581DDDA83A9A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:29:49.598121122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T02:29:50.643722772Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE581E8837FBF7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T02:29:54.597809037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:29:59.598096669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:30:04.597583348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:30:09.597919046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:30:14.596972591Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:30:14.597005869Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:30:14.598078168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:30:19.597328825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:30:24.597252703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:30:29.598134567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:30:34.597344148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:30:36.294234957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:30:36.305869582Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE582929E16746, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:30:39.59735225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:30:44.578313936Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:30:44.597606786Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:30:44.597635556Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:30:44.597634613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:30:49.597885973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:30:54.597244397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:30:59.597604296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:31:04.597930376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:31:09.597305022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:31:14.596957437Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:31:14.596992747Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:31:14.598085177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:31:19.597424627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:31:24.597596441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:31:26.661243231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:31:26.670595507Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5834E3DB58C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:31:29.59824871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:31:34.598200859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:31:39.597512993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:31:44.578587839Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:31:44.597503674Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:31:44.597529662Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:31:44.597530984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:31:49.598027349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:31:54.597733956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:31:59.597741416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:32:04.598014677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:32:09.597450392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:32:14.597141264Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:32:14.597172536Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:32:14.598256499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:32:14.604007819Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-07-02T02:32:14.605817817Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008274094s
level=warn ts=2026-07-02T02:32:19.597455807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:32:24.59731956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:32:24.597371237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:32:24.609471182Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE58426147EB9C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:32:29.5982918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:32:34.598161809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:32:39.597608064Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:32:44.578349792Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:32:44.59730124Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:32:44.597311836Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T02:32:44.597329588Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:32:44.597380518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:32:44.597465718Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-07-02T02:32:44.597474123Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T02:32:44.597493207Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.25µs success=true
level=warn ts=2026-07-02T02:32:49.597532289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:32:54.597345239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:32:59.597995987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:33:04.597946803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:33:07.375978534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:33:08.668728858Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE584CA36B93A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:33:08.668752673Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T02:33:08.668778177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:33:08.677456581Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE584CA3F1F936, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:33:09.597587862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:33:14.597022122Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:33:14.597054677Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:33:14.598125275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:33:19.597773417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:33:22.513885277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:33:22.525501728Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE584FDD59C413, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:33:24.597268416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:33:29.59798052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:33:34.598236767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:33:39.598287099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T02:33:43.455926974Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE5854BCE9DE27, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T02:33:44.57832206Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:33:44.59697479Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:33:44.597014646Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:33:44.598113874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:33:47.445811738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:33:47.455218632Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5855AB460F0B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:33:49.597452617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:33:54.597650445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:33:59.5976243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:34:04.597706492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:34:09.597710275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:34:14.597718943Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T02:34:14.597742965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:34:14.5977538Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:34:19.597262918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:34:24.598064354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:34:29.597374354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:34:34.597325531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:34:36.463634585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:34:37.266017181Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5861443B66BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:34:39.597517897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:34:44.579266569Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:34:44.597201662Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:34:44.597229477Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:34:44.597264309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:34:49.598188088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:34:54.597500354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:34:59.597979053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:35:04.597296217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:35:09.597463017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:35:14.59718413Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:35:14.597214603Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:35:14.597252541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:35:19.597743111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:35:24.597851757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:35:29.598286196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:35:31.050451399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:35:31.061679134Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE586DCAB3CE23, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:35:34.597920549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:35:39.597672247Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:35:44.578779772Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T02:35:44.597510408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:35:44.597532508Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:35:44.597555849Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:35:49.598154555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:35:54.597373031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:35:59.598207059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:36:04.598262391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:36:09.597779991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:36:14.597261302Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:36:14.597302054Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:36:14.597333817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:36:19.597913021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:36:24.597717382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:36:29.434053316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:36:29.445610476Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE587B62A7D099, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:36:29.598156549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:36:34.597928599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:36:39.598262744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:36:44.578627437Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:36:44.597661177Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:36:44.597692408Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:36:44.597699191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:36:49.598097402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:36:54.59783599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:36:59.598282428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:37:04.597931028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:37:09.59776438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:37:14.597283864Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:37:14.597318993Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:37:14.597349223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:37:14.604020615Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-07-02T02:37:14.605790375Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.0081678s
level=warn ts=2026-07-02T02:37:19.59811953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:37:22.446711494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:37:22.458613506Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5887BA794373, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:37:24.598066262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:37:29.59753699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:37:34.597936167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:37:39.59780266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:37:44.578317772Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:37:44.597785037Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T02:37:44.597798616Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:37:44.59783044Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:37:44.597843143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:37:44.597903767Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-07-02T02:37:44.597912339Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T02:37:44.597928125Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.847µs success=true
level=warn ts=2026-07-02T02:37:49.598080501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T02:37:51.603261945Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE588E83A506A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T02:37:54.597594341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:37:59.597372983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:38:04.597415662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:38:09.597977227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:38:13.488021057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:38:13.498579263Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE58939CB13D81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T02:38:14.59745125Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:38:14.597481519Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:38:14.597472209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:38:19.597593126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:38:24.597527863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:38:29.597377456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:38:34.597262166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:38:39.598072821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:38:44.5783129Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:38:44.597110764Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:38:44.597136996Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:38:44.598198921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:38:49.598156814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:38:54.597360232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:38:59.597628363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:39:04.598017069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:39:09.597604693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:39:11.507267812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:39:11.517270239Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE58A11EE1477D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T02:39:14.597895978Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:39:14.597928523Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:39:14.597918983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:39:19.598006302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:39:24.598032875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:39:29.597814363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:39:34.59796602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:39:39.597603856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:39:44.578724758Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:39:44.597108056Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:39:44.597140709Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:39:44.598199061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:39:49.598262424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:39:54.598185392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:39:58.82845392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:39:58.837959919Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE58AC2369438B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:39:58.837980832Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T02:39:58.838004311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:39:58.846474479Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE58AC23ECED96, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:39:59.597914915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:40:04.597706116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:40:09.597571624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:40:14.597652301Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T02:40:14.597664078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:40:14.597684948Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T02:40:17.917942693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:40:17.928812767Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE58B095504F17, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:40:19.597481344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:40:24.597999673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:40:29.597505436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:40:34.597585827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:40:38.117554165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:40:38.128257463Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE58B5494BECA4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:40:39.59764368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:40:44.578390986Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:40:44.597630197Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:40:44.597662792Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:40:44.597690095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:40:49.59750535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:40:54.59725518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:40:59.597693864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:41:04.597275177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:41:09.597662958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:41:14.597930889Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:41:14.597961532Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:41:14.597958142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:41:19.59814213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:41:24.598144263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:41:29.597625345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:41:33.943984154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:41:33.95512548Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE58C248D642FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:41:34.598120899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:41:39.598237627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:41:44.578853218Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T02:41:44.597690337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:41:44.597703236Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:41:44.597732643Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:41:49.598071339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T02:41:54.318778082Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE58C7069D6C7C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T02:41:54.597579237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:41:59.598300835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:42:04.597397781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:42:09.598242694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:42:14.597374924Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:42:14.597421119Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:42:14.597420241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:42:14.60395922Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-07-02T02:42:14.605580876Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007529963s
level=info ts=2026-07-02T02:42:17.798817849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:42:17.827673735Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE58CC7FD7D779, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:42:19.597484512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:42:24.59778287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:42:29.597688274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:42:34.598168723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:42:39.598033391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:42:44.578365605Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:42:44.597380171Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-07-02T02:42:44.597390079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:42:44.59743267Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:42:44.597457979Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T02:42:44.597523544Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-07-02T02:42:44.597532713Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T02:42:44.597550582Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.631µs success=true
level=warn ts=2026-07-02T02:42:49.598100218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:42:54.598311451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:42:59.597981005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:43:04.597445196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:43:09.366266377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:43:09.377599845Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE58D88072A4D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:43:09.597879647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:43:14.596949437Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:43:14.596980791Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:43:14.598046542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:43:19.597752743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:43:24.597301533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:43:29.597540304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:43:34.597400812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:43:39.598219839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:43:44.578518269Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:43:44.597086166Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:43:44.597115185Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:43:44.598198759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:43:49.598274195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:43:54.597291705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:43:59.597955237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:43:59.94895939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:43:59.960716061Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE58E4476F218E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:44:04.598118896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:44:09.597999599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:44:14.597827605Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T02:44:14.597836875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:44:14.597862005Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:44:19.597732195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:44:24.597360189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:44:29.597894554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:44:34.597712914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:44:39.598103518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:44:44.159065629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:44:44.170485382Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE58EE928E0A06, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T02:44:44.578391246Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:44:44.597864826Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T02:44:44.597866799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:44:44.597890956Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:44:49.598045413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:44:54.598058819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:44:59.598136446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:45:04.59812351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:45:09.597399484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:45:14.597132282Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:45:14.597165597Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:45:14.598266583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:45:19.597419282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:45:24.598163849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:45:29.597508841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:45:34.597458542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:45:39.597515062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T02:45:42.353632438Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE58FC1E8D51DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T02:45:43.01209108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:45:43.022027105Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE58FC466140B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T02:45:44.578313312Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:45:44.597036353Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:45:44.597061844Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:45:44.598142275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:45:49.598118186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:45:54.597992058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:45:59.597567178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:46:04.597532563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:46:09.597478336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:46:14.597141073Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:46:14.59717359Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:46:14.598241113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:46:19.597842418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:46:24.598289539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:46:29.597254037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:46:30.733615414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:46:30.744779436Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE590762DF1BBC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:46:30.744801985Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T02:46:30.744830142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:46:30.753507735Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59076367B1E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:46:34.597778944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:46:39.597648142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:46:44.578316802Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:46:44.597626579Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T02:46:44.597640735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:46:44.597652429Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T02:46:45.94276512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:46:46.225471151Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE590AFD993B85, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:46:49.597993677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:46:54.59812942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:46:59.598253122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:47:04.597997294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:47:09.597527534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:47:14.59731477Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:47:14.597347184Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:47:14.597341464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:47:14.603513347Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-07-02T02:47:14.605113428Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007442341s
level=info ts=2026-07-02T02:47:18.786447879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:47:18.795915528Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE591292F17A59, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:47:19.598106214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:47:24.597303895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:47:29.597806012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:47:34.598118867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:47:39.597782195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:47:44.578373257Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:47:44.597248572Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T02:47:44.597269719Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:47:44.597295466Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:47:44.597305783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:47:44.597390482Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-07-02T02:47:44.597398789Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T02:47:44.597435729Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.032µs success=true
level=warn ts=2026-07-02T02:47:49.597899766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:47:54.59782416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:47:59.598001339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:48:00.747624632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:48:00.760696734Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE591C583D91B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:48:04.597548104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:48:09.597306418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:48:14.597490247Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:48:14.597526493Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:48:14.597529408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:48:19.597666459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:48:24.597276505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:48:29.542316695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T02:48:29.549339823Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59230C2DB18E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T02:48:29.598034846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:48:29.842643162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="153 B" avg_comp="153 B" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T02:48:29.845092737Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59231DCF1E9D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T02:48:30.143392621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T02:48:30.14575582Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59232FBB49C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T02:48:30.442908239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T02:48:30.445322837Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5923419616A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T02:48:30.74337407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="30 kB" avg_uncomp="30 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T02:48:30.745866126Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5923537E2EB5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T02:48:31.042437467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="134 B" avg_comp="134 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T02:48:31.04484612Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE592365523A72, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T02:48:31.342639901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="47 kB" avg_uncomp="47 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T02:48:31.345058211Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5923773756C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T02:48:31.643032584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="101 B" avg_uncomp="101 B" idle=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T02:48:31.645370656Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5923891D9BEB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T02:48:31.943278055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" idle=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T02:48:31.945594912Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59239B02D2C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-02T02:48:32.242463767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="193 B" avg_comp="193 B" total_uncomp="274 B" avg_uncomp="274 B" idle=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T02:48:32.244701296Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5923ACD70A1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T02:48:32.542453147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T02:48:32.544774406Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5923BEB8B82B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-07-02T02:48:34.597386696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:48:39.598113741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:48:41.220359165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T02:48:41.222650938Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5925C3F71152, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T02:48:42.005104296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T02:48:42.00733437Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5925F2BC3530, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T02:48:43.127243915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T02:48:43.129699692Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE592635A22F45, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T02:48:43.233062666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T02:48:43.235356615Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59263BEEA89B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T02:48:43.344529499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T02:48:43.346711577Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59264292D17F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T02:48:44.579197414Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T02:48:44.597922354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:48:44.597931718Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:48:44.597953777Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T02:48:44.758528909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T02:48:44.760993542Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE592696DDF6A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T02:48:44.912872575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T02:48:44.915251029Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5926A00F5CFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T02:48:47.360314774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T02:48:47.362639236Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE592731F01FB3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T02:48:49.597891646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:48:50.401585364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T02:48:50.40389307Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5927E7356085, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T02:48:51.350522462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T02:48:51.352797039Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59281FC54394, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T02:48:51.694042018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T02:48:51.69637347Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5928343F17BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-02T02:48:54.597772806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:48:57.000057092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:48:57.007586961Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE592970D121AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:48:59.597508523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:49:03.973506732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T02:49:03.978337158Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE592B104F69D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T02:49:04.27406007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="149 kB" avg_uncomp="149 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T02:49:04.27666928Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE592B2216DFBC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T02:49:04.597360682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:49:07.289301516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T02:49:07.291621183Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE592BD5CC508D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T02:49:07.496054914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T02:49:07.49836504Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE592BE21F438F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T02:49:08.646757748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T02:49:08.649052067Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE592C26B54827, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T02:49:09.598149077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:49:14.597159064Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:49:14.597190091Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:49:14.598251859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:49:15.546623181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T02:49:15.549164213Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE592DC1FB4B21, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T02:49:15.588573934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T02:49:15.590842733Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE592DC47853F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T02:49:16.9296862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T02:49:16.931976444Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE592E1467BC02, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T02:49:18.035025287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T02:49:18.037304555Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE592E564AB09E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T02:49:18.342110403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T02:49:18.34445569Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE592E68993F7B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T02:49:19.230209602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T02:49:19.232561269Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE592E9D87F098, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T02:49:19.598304561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:49:19.701580621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T02:49:19.703889192Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE592EB99F17BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-02T02:49:21.700183479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T02:49:21.702605415Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE592F30C0A467, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T02:49:22.911320294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T02:49:22.913760916Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE592F78F2AEE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T02:49:24.598026232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:49:28.047312374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T02:49:28.049732743Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5930AB133321, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T02:49:28.486120356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T02:49:28.48849521Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5930C53AD28A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T02:49:29.598002753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:49:34.598009683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:49:38.102306677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T02:49:38.104693886Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE593302656610, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T02:49:38.71170841Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE5933269710C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T02:49:38.712693228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T02:49:38.714615539Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE593326C2838D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T02:49:39.597599027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:49:41.881677448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:49:41.888534085Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5933E3EDAE43, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T02:49:44.578379636Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:49:44.597043584Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:49:44.597070862Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:49:44.598168539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:49:47.55745386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T02:49:47.562309393Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5935361CF0D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T02:49:47.607747952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="513 B" avg_comp="513 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T02:49:47.610226503Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE593538F7CE92, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T02:49:49.597945021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:49:54.597494417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:49:56.62453669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T02:49:56.626996986Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59375269460A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T02:49:58.480097691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T02:49:58.482504527Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5937C10282D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-02T02:49:59.597535703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:50:00.562844685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T02:50:00.565527521Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59383D287DF2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-02T02:50:00.857689222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T02:50:00.860095101Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59384EB9F5D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T02:50:03.445042459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T02:50:03.447385691Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5938E8F18D5F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T02:50:04.597901318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:50:05.674582002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T02:50:05.676926522Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59396DD4CD3B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T02:50:06.656621717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T02:50:06.659045172Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5939A85D99BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T02:50:09.219825744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T02:50:09.222183371Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE593A4125A418, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T02:50:09.396771118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T02:50:09.399219879Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE593A4BB16ECA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T02:50:09.598076978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:50:14.597623814Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:50:14.597653052Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:50:14.597651168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:50:15.407819873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T02:50:15.410307486Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE593BB1FCB4AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T02:50:19.59734275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:50:22.382361114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T02:50:22.384589322Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE593D51B0256A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T02:50:24.598058004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:50:29.597521966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:50:30.555736686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T02:50:30.558201838Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE593F38DF313F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T02:50:34.597871387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:50:37.248351317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T02:50:37.250758571Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5940C7C6C3E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T02:50:39.226336371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:50:39.233260321Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59413DF21ED1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:50:39.597713368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:50:40.336246596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T02:50:40.342890207Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59418013CFF6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T02:50:42.00372727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T02:50:42.006018742Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5941E3371504, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T02:50:43.921006526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="630 B" avg_comp="630 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T02:50:43.92327245Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5942557EC033, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-02T02:50:44.579259693Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:50:44.597513477Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T02:50:44.597513073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:50:44.597537089Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T02:50:49.168648663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T02:50:49.171013959Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59438E489837, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T02:50:49.597976216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:50:54.144608475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T02:50:54.147049613Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5944B6DFE8F0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T02:50:54.597264262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:50:59.59800503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:50:59.727302833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T02:50:59.729763664Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE594603A19C6A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T02:51:00.016589984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="663 B" avg_comp="663 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T02:51:00.019070549Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE594614DE9D47, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T02:51:00.616294138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T02:51:00.618650403Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5946389C9AB7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T02:51:02.297549844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T02:51:02.299979785Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59469CD2D039, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T02:51:03.782245681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T02:51:03.784601068Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5946F5512380, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T02:51:04.597436912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:51:06.152222141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T02:51:06.154760391Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59478296CA64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T02:51:09.584282143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T02:51:09.586598976Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59484F259FD3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T02:51:09.597647725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:51:14.597206905Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:51:14.597238808Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:51:14.597307754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:51:19.597510306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:51:20.008731152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T02:51:20.012765259Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE594ABC95E0E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T02:51:22.024389041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T02:51:22.026937982Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE594B34A33CDB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T02:51:24.360953786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:51:24.367904543Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE594BC02D080C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:51:24.598266876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:51:29.597875466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:51:30.119453714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T02:51:30.12425873Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE594D17483EC5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T02:51:34.597815267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:51:39.598286057Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:51:39.910068325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T02:51:39.912585109Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE594F5EB5BA20, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-02T02:51:40.243046963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T02:51:40.245443473Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE594F728E33DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T02:51:43.592919591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T02:51:43.595621963Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59503A3B9931, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T02:51:44.578377077Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:51:44.597800223Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:51:44.597842016Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:51:44.597846497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:51:47.608215945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T02:51:47.610685818Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5951298EABB6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T02:51:48.864066396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T02:51:48.86651254Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5951746934C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T02:51:49.597457619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:51:54.597869519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:51:55.425551749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T02:51:55.428028138Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5952FB8039C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T02:51:56.694710042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T02:51:56.697151145Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59534726D353, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T02:51:58.255464044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T02:51:58.257840343Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5953A42D23A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T02:51:58.68633025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="314 B" avg_comp="314 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T02:51:58.688774064Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5953BDDC8B5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T02:51:59.464851637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T02:51:59.467267643Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5953EC4470D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T02:51:59.562515871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T02:51:59.564920717Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5953F2150AE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T02:51:59.598033965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:52:04.597949285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:52:08.805819194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T02:52:08.80826509Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59561908497D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T02:52:09.598165271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:52:14.59758688Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:52:14.597617268Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:52:14.597661349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:52:14.603139789Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-07-02T02:52:14.604761175Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007197304s
level=info ts=2026-07-02T02:52:14.71117095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T02:52:14.713615496Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE595779030413, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T02:52:18.487670936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T02:52:18.490053055Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59585A1BC13D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T02:52:19.597588287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:52:20.924381116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T02:52:20.926760751Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5958EB597497, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T02:52:21.498065053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:52:21.505021351Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59590DD12C0E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T02:52:24.144582659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T02:52:24.14939255Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5959AB6EBC81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-02T02:52:24.598111431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:52:29.597272449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:52:29.995346543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T02:52:29.997688367Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE595B08040D7C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T02:52:34.59773272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:52:39.59829424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:52:40.239316562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="920 B" avg_comp="920 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T02:52:40.241715358Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE595D6A9CC2D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T02:52:40.618854908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T02:52:40.62126783Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE595D813BABA9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T02:52:41.808886633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T02:52:41.811302297Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE595DC82ABD25, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T02:52:42.676199494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T02:52:42.678525091Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE595DFBDB2AF0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T02:52:43.857865539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T02:52:43.860285481Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE595E424AF8CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T02:52:44.578343486Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:52:44.597492264Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-07-02T02:52:44.597516662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:52:44.597529619Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:52:44.597558419Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T02:52:44.597623132Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-07-02T02:52:44.597630958Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T02:52:44.597647787Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.893µs success=true
level=info ts=2026-07-02T02:52:46.49067045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T02:52:46.493087355Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE595EDF38B223, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T02:52:47.192099337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T02:52:47.194371743Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE595F0905C19B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T02:52:47.251439722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T02:52:47.253744786Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE595F0C8ED503, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T02:52:49.597957012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:52:54.597260032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:52:55.191585686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T02:52:55.194014242Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5960E5D6EDE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T02:52:59.597437166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:53:03.637220004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T02:53:03.639684009Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5962DD3BF8E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T02:53:04.59785318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:53:04.953008707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T02:53:04.955472647Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59632BAA9A06, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-02T02:53:05.659503209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T02:53:05.661992464Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE596355C5B105, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T02:53:09.598043277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:53:10.085916325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T02:53:10.088254582Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59645D9A6D24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T02:53:14.596975611Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:53:14.597009954Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:53:14.598086841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:53:19.598131375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:53:20.621813707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:53:22.478354363Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5967401C1AC1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:53:22.478377135Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T02:53:22.47842309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:53:22.488695659Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE596740BAD734, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T02:53:24.463048556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T02:53:24.468174725Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5967B6B5CCAA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T02:53:24.597591017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:53:25.000583052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="146 B" avg_comp="146 B" total_uncomp="21 kB" avg_uncomp="21 kB" idle=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T02:53:25.002914958Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5967D69560FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T02:53:26.428350838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T02:53:26.430881609Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59682BB264D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T02:53:27.760965721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T02:53:27.763457088Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59687B203643, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T02:53:29.597912579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:53:30.633316114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T02:53:30.635785583Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59692654DC16, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T02:53:32.917740456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:53:32.924541869Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5969AEC038BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:53:33.829058568Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE5969E4AD45A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T02:53:34.59726812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:53:36.473011084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T02:53:36.477748439Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE596A8288E82C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T02:53:37.229449883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T02:53:37.231897215Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE596AAF7D8C1E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T02:53:37.969085524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T02:53:37.971510477Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE596ADB91D9F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T02:53:38.020658489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T02:53:38.022841203Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE596ADEA35F92, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T02:53:39.320275306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T02:53:39.32260281Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE596B2C1B5390, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T02:53:39.598125498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:53:40.052345046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T02:53:40.054893814Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE596B57BFA3E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T02:53:44.57836611Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:53:44.597556054Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:53:44.597583905Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:53:44.597579524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:53:46.778383878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T02:53:46.78079508Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE596CE8A548B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T02:53:47.53659485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T02:53:47.538954672Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE596D15D6EB1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T02:53:47.562005658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T02:53:47.564133749Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE596D17580C3E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-02T02:53:49.59794286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:53:53.036877777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T02:53:53.039230119Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE596E5DAE16BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T02:53:54.597304994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:53:56.155075851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:53:56.161963472Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE596F17CE66AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T02:53:58.531752683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T02:53:58.538468698Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE596FA575C82A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T02:53:59.59781368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:54:04.598263348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:54:06.017598687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T02:54:06.020031854Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59716364CCB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-02T02:54:09.597932218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:54:12.433346487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T02:54:12.435716153Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5972E1CC7D17, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T02:54:14.597346309Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T02:54:14.597348897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:54:14.5973758Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T02:54:15.412806695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T02:54:15.415272878Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59739365BEBC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T02:54:18.172921241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T02:54:18.175335132Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE597437E86286, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T02:54:19.079065776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T02:54:19.081564632Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59746DEB912A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T02:54:19.598350506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:54:20.155363611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="513 B" avg_comp="513 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T02:54:20.157817252Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5974AE1278FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T02:54:20.157831625Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T02:54:20.157850281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T02:54:20.159885685Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5974AE3377AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T02:54:20.691803573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T02:54:20.696258281Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5974CE2A4335, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T02:54:22.865285839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T02:54:22.871700251Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59754FD2F98E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T02:54:24.49580193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T02:54:24.498073795Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5975B0C6C84C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T02:54:24.597320792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:54:29.597444389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:54:29.982213217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T02:54:29.984423132Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5976F7C988A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T02:54:32.850895464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T02:54:32.853301147Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5977A2C88DE2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T02:54:34.094602509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T02:54:34.096957958Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5977ECE9DA8E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T02:54:34.597581586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:54:35.146615447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T02:54:35.148921429Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59782B9D49CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T02:54:35.927661467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T02:54:35.932057283Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59785A4B8962, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T02:54:35.980372831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T02:54:35.986642143Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59785D8B907C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T02:54:39.597864757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:54:43.003449993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T02:54:43.005928573Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5979FFEBF682, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T02:54:44.032853146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T02:54:44.035457958Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE597A3D48C56A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-07-02T02:54:44.035480782Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T02:54:44.03550445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T02:54:44.037362642Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE597A3D69EDB5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-02T02:54:44.578316789Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:54:44.597048027Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:54:44.597082Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:54:44.598205803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:54:45.017559713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:54:45.025375782Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE597A784AA65E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T02:54:45.692446836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T02:54:45.697052192Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE597AA0554E27, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T02:54:49.597876667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:54:54.597570821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:54:54.70187551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T02:54:54.704271619Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE597CB9344A5E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T02:54:55.22498823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T02:54:55.227251817Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE597CD860B04A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T02:54:55.227267336Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T02:54:55.227287439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T02:54:55.229110609Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE597CD87F1A14, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T02:54:59.291099131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T02:54:59.293516358Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE597DCABE8BE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-02T02:54:59.597888158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:55:01.979446275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T02:55:01.981805315Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE597E6AFAFBFF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T02:55:01.981823706Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T02:55:01.981841852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T02:55:01.98361051Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE597E6B188E02, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T02:55:04.597341456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:55:09.597985119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:55:10.84068809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T02:55:10.843126692Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59807B269920, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T02:55:11.797012297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T02:55:11.799578278Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5980B4298519, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-02T02:55:11.79959381Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T02:55:11.799611906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T02:55:11.801343906Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5980B447B030, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T02:55:13.682918103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T02:55:13.68755175Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE598124B05953, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T02:55:13.7635835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T02:55:13.769996531Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5981299B4833, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-02T02:55:13.770014815Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T02:55:13.770035799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T02:55:13.771855053Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE598129BAEA93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T02:55:13.773924011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T02:55:13.775663311Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE598129F503BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T02:55:13.775677904Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T02:55:14.070399048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T02:55:14.072729743Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59813BA75F70, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T02:55:14.597299769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:55:14.597321147Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:55:14.597345073Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T02:55:15.030229884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T02:55:15.032608912Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE598174DDFE01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T02:55:15.032623705Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T02:55:15.03264143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T02:55:15.0342969Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE598174FAE02B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T02:55:15.615973265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T02:55:15.618310117Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE598197C74458, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-07-02T02:55:15.618325262Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T02:55:15.618342665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T02:55:15.620112788Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE598197E55806, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T02:55:16.287829867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T02:55:16.290170953Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5981BFD343F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T02:55:16.290186304Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T02:55:16.290203205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T02:55:16.292020696Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5981BFF1E42E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T02:55:19.597769541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:55:20.549722696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T02:55:20.552298743Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5982BDDE3AC4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T02:55:20.662763604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T02:55:20.665223794Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5982C4979875, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T02:55:24.598100539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:55:25.251786432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T02:55:25.254192527Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5983D61E7F8E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T02:55:25.619383576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T02:55:25.621599552Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5983EC05F400, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T02:55:25.621616457Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T02:55:25.621634061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T02:55:25.623284018Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5983EC2265A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T02:55:25.730564586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T02:55:25.732870113Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5983F2A6A963, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T02:55:25.787128234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T02:55:25.789558511Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5983F60657C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T02:55:26.38506923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T02:55:26.387340041Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE598419A9C350, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T02:55:28.790366158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T02:55:28.792792956Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5984A9095363, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T02:55:29.597810923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:55:29.894953697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T02:55:29.897468769Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5984EADFBBD0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T02:55:29.897485387Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T02:55:29.897504744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T02:55:29.89929534Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5984EB0009A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T02:55:30.18582821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T02:55:30.188341714Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5984FC36D941, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-02T02:55:30.286573921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T02:55:30.288976019Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE598502370949, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T02:55:31.624800741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:55:31.632120436Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE598552463B6F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T02:55:33.492088938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T02:55:33.498814162Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5985C1896116, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T02:55:34.597895205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:55:39.597589134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:55:41.692390835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T02:55:41.694749557Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5987AA0D5157, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T02:55:42.388864567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T02:55:42.391198659Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5987D3907764, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T02:55:42.391215727Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T02:55:42.391235554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T02:55:42.393010498Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5987D3AF76F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T02:55:43.053843204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T02:55:43.056104017Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5987FB331C16, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T02:55:43.168275511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T02:55:43.170761726Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59880207026D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T02:55:44.579065377Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:55:44.596033412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T02:55:44.597328328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:55:44.597335719Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:55:44.597358497Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=error ts=2026-07-02T02:55:44.598223808Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5988571D3A9D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T02:55:48.991424166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T02:55:48.993725664Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59895D1B051C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-07-02T02:55:49.59775486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:55:49.879340406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T02:55:49.881937507Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5989920CA111, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T02:55:54.353910903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T02:55:54.356430209Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE598A9CBEA45D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T02:55:54.597855287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:55:56.403268708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T02:55:56.405689407Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE598B16E479EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T02:55:56.52588386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T02:55:56.530394803Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE598B1E53AE26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T02:55:58.486666026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T02:55:58.493074288Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE598B93500FB1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T02:55:59.157772639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T02:55:59.160112968Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE598BBB122C4F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T02:55:59.597440954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:56:01.9804356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T02:56:01.982948799Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE598C63530EE2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T02:56:04.597930388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:56:09.598128955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:56:09.773843326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T02:56:09.776267907Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE598E33D6CD17, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T02:56:12.606713574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T02:56:12.60916836Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE598EDCB1F043, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T02:56:14.597600583Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:56:14.5976331Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:56:14.597706748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:56:15.943370061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="663 B" avg_comp="663 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T02:56:15.945765537Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE598FA392E2D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-02T02:56:15.945782366Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T02:56:15.945800841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T02:56:15.947676794Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE598FA3B1F83E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T02:56:16.252586612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T02:56:16.254962638Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE598FB5FF650B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T02:56:19.597955185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:56:20.005109353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T02:56:20.0074388Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE599095AB07C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T02:56:22.485759377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T02:56:22.488070965Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59912986C096, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T02:56:24.597721198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:56:25.55205455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T02:56:25.554661Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5991E04E1044, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T02:56:26.623957574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T02:56:26.626365181Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5992202FB2ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T02:56:26.861814297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="630 B" avg_comp="630 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T02:56:26.864196104Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59922E5D2DBE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-02T02:56:27.977361727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T02:56:27.979679147Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE599270DA11F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T02:56:29.597710541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:56:31.462387054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:56:31.469254456Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE599340D94F30, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:56:34.597829493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:56:39.17088439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T02:56:39.177395564Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59950C486A3E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T02:56:39.598126093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:56:43.826905587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T02:56:43.829321058Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5996219046EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T02:56:44.578331207Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:56:44.596971792Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:56:44.597008403Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:56:44.598078757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:56:45.685915813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T02:56:45.68825741Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5996905D3BA5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T02:56:48.343523453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T02:56:48.345972944Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59972EC5F4EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T02:56:48.509511491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T02:56:48.511899641Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE599738AA7EFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-07-02T02:56:49.598183937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:56:51.145588602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T02:56:51.149951158Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5997D5E81B82, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T02:56:54.597762255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:56:55.917947291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T02:56:55.92373328Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5998F27241CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T02:56:56.265628925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="314 B" avg_comp="314 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T02:56:56.267995192Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE599906F68B3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T02:56:59.597289919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:57:00.922940561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="920 B" avg_comp="920 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T02:57:00.925245758Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE599A1C901E99, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T02:57:02.889235749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T02:57:02.891534092Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE599A91C2A726, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T02:57:04.597879851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:57:07.168857532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T02:57:07.171282098Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE599B90D97703, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T02:57:09.598111534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:57:10.530630618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T02:57:10.532946178Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE599C5938546D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T02:57:10.688600459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T02:57:10.690941395Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE599C62A38198, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-07-02T02:57:13.298526836Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE599CFE133B70, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T02:57:14.597788867Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:57:14.597820862Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:57:14.597863162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:57:14.602890219Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-07-02T02:57:14.604707901Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006948704s
level=info ts=2026-07-02T02:57:17.994862674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:57:18.001572668Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE599E1663EC07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T02:57:19.571245672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T02:57:19.575731948Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE599E743692F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T02:57:19.598128357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:57:19.828264975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T02:57:19.830509538Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE599E836636B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T02:57:24.588278917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T02:57:24.590398939Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE599F9F1E3FA4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T02:57:24.597543521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:57:27.218435962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T02:57:27.220829295Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59A03BE56D86, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T02:57:29.597791321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:57:33.887578866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T02:57:33.889935921Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59A1C96876FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T02:57:34.59736316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:57:34.639481368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T02:57:34.641756419Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59A1F6380B38, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T02:57:34.975345816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T02:57:34.977672564Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59A20A3D7EB9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T02:57:39.597809787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:57:41.782560707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T02:57:41.784923647Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59A39FFBF0FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T02:57:44.579033704Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:57:44.597124212Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T02:57:44.597149528Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:57:44.597176116Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T02:57:44.597247709Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-07-02T02:57:44.597264583Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=warn ts=2026-07-02T02:57:44.597278472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:57:44.597283278Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.273µs success=true
level=info ts=2026-07-02T02:57:44.603340739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T02:57:44.605569463Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59A4481BA3FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T02:57:44.675900948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T02:57:44.67824592Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59A44C702CBD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T02:57:45.81304479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T02:57:45.816001125Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59A4903CAC56, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T02:57:49.545456648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T02:57:49.549872693Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59A56ED0665F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T02:57:49.598205476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:57:52.797593102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T02:57:52.803127291Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59A630B9FA48, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T02:57:54.598150538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:57:58.567271421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T02:57:58.569675494Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59A7886F18B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T02:57:59.59784867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:58:02.283957293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T02:58:02.28627075Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59A865F6D6C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T02:58:04.597881098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:58:06.712849177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T02:58:06.71514316Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59A96DF27DE5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T02:58:06.899671096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T02:58:06.902000761Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59A9791406EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-02T02:58:08.337816904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T02:58:08.340249209Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59A9CECEA3EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-02T02:58:09.597302902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:58:10.089979947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T02:58:10.092285587Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59AA373CDEBB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T02:58:10.461595541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:58:10.468301324Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59AA4DA6F084, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T02:58:11.481272278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T02:58:11.487683973Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59AA8A674C39, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T02:58:14.597505778Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:58:14.597534225Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:58:14.597531898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:58:15.944654017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T02:58:15.947061319Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59AB9433FF91, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T02:58:17.309213788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T02:58:17.311623631Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59ABE589B91D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T02:58:19.597394656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:58:21.037813025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T02:58:21.040189501Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59ACC3C8E80F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T02:58:24.597820704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:58:26.382865541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T02:58:26.385306061Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59AE025F1154, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T02:58:29.597454573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:58:34.535704527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T02:58:34.540300676Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59AFE872E107, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T02:58:34.597518793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:58:38.156277756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T02:58:38.16235063Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59B0C057B6D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T02:58:39.597876622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:58:41.135703731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T02:58:41.138205949Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59B171B6C80B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T02:58:42.858204986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T02:58:42.860581228Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59B1D8609CB3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T02:58:43.52694783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T02:58:43.529582152Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59B2003F8F83, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T02:58:44.578508437Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:58:44.597057538Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:58:44.597084956Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:58:44.598158071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:58:45.071555337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T02:58:45.073907776Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59B25C4D6F77, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T02:58:47.040829468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T02:58:47.043266568Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59B2D1AF8E6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T02:58:49.597666798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:58:50.281597424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T02:58:50.283918659Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59B392D84D08, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T02:58:50.43506646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T02:58:50.437530427Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59B39BFEABD6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T02:58:53.681671008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T02:58:53.684189651Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59B45D828E21, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-02T02:58:54.597861828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:58:58.550538578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T02:58:58.55285646Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59B57FB671C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T02:58:58.841827972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T02:58:58.844111493Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59B591128DA4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T02:58:59.597989723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:59:01.790513232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T02:59:01.792907359Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59B640D56A60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T02:59:04.597766294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:59:09.337113673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T02:59:09.344055143Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59B802EA2BC0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T02:59:09.59744954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:59:12.363454744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T02:59:12.369400002Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59B8B73E288C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T02:59:14.597903637Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:59:14.597937124Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:59:14.597971413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:59:16.100743418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T02:59:16.103094548Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59B995CA0319, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T02:59:17.074900002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T02:59:17.077198332Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59B9CFD98875, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T02:59:19.598236035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:59:24.597308684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:59:24.609504134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T02:59:24.613908483Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59BB9111F5E7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T02:59:25.895304957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T02:59:25.89981211Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59BBDDB6DA1F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T02:59:25.89983058Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T02:59:25.899850388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T02:59:25.901776893Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59BBDDD7BA06, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T02:59:27.514393856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T02:59:27.516903659Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59BC3E18C7FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T02:59:28.46020288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T02:59:28.462546513Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59BC76781CE8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T02:59:29.210817429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T02:59:29.213107529Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59BCA3356B0D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T02:59:29.597486116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:59:34.597884849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:59:37.299074513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="513 B" avg_comp="513 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T02:59:37.301419191Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59BE854E803C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T02:59:38.66205851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T02:59:38.664426057Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59BED68B041A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-07-02T02:59:39.597767239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:59:41.226109826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T02:59:41.228437278Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59BF6F5F4EF2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T02:59:41.627529347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T02:59:41.629802488Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59BF874CBAEF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T02:59:43.656298074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T02:59:43.658712914Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59C0003AA84B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T02:59:43.759937258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T02:59:43.762114131Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59C006658E6E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-02T02:59:44.578313537Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T02:59:44.59281274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T02:59:44.594990694Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59C0380991B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T02:59:44.596996336Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T02:59:44.597018795Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T02:59:44.598127122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:59:49.598103045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T02:59:52.917740444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T02:59:52.920305595Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59C22842F8B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T02:59:52.979628659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T02:59:52.981976605Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59C22BEFB3AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T02:59:54.598255491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T02:59:59.597608357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:00:00.357511601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:00:00.35982509Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59C3E3B07DA4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:00:00.524388398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:00:00.531877285Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59C3EDF29D15, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:00:00.531898021Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:00:00.53192134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:00:00.543292162Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59C3EE9D59FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T03:00:04.597899346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:00:04.60602825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:00:04.611146349Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59C4E1153660, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:00:05.122126864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:00:05.12478369Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59C4FFB22BDD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T03:00:09.597969564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:00:10.864054537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:00:10.871396493Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59C6563C332F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:00:12.411266547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:00:12.415793304Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59C6B248A256, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:00:14.597653829Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:00:14.597690453Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:00:14.597770625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:00:18.208852171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:00:18.213873375Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59C80BDF30DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T03:00:19.597461741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:00:21.37024659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:00:21.375019097Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59C8C84B8623, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T03:00:24.598301931Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:00:24.736613113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:00:24.738885378Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59C990CCDAF4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:00:25.745390454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:00:25.747834999Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59C9CCEF4BE6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:00:29.597725366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:00:30.012833206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T03:00:30.015146262Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59CACB4A0D81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T03:00:31.678975545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:00:31.681338832Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59CB2E996D29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:00:34.507256452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:00:34.509729871Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59CBD72F2E47, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:00:34.509747066Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:00:34.509764346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:00:34.511547481Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59CBD74DE4AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T03:00:34.59812372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:00:36.166160839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T03:00:36.168621109Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59CC3A0F7A47, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T03:00:36.814656239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:00:36.817211849Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59CC60B7DBCE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T03:00:36.915440686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="630 B" avg_comp="630 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T03:00:36.917706991Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59CC66B6C66D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-02T03:00:39.597908116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:00:39.66917007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="87 kB" avg_uncomp="87 kB" idle=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:00:39.671555748Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59CD0ADAAC46, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:00:39.779669527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:00:39.78696386Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59CD11BC5C08, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:00:39.969392043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="421 kB" avg_uncomp="421 kB" idle=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:00:39.975704412Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59CD1CF912B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=info ts=2026-07-02T03:00:40.269574064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="126 B" avg_comp="126 B" total_uncomp="10 kB" avg_uncomp="10 kB" idle=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:00:40.272240388Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59CD2EA65E28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:00:42.519379482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="663 B" avg_comp="663 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:00:42.521686544Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59CDB4BCACE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:00:43.965721095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:00:43.968061745Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59CE0AF2AAF6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:00:44.579123906Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:00:44.597504594Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:00:44.597533319Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:00:44.597530604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:00:47.106805572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:00:47.109053767Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59CEC62B50FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:00:48.62911245Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE59CF20C6DE00, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T03:00:49.597845622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:00:50.113538561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:00:50.116295216Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59CF7966EDD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:00:50.779009241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:00:50.781365796Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59CFA10CC4D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:00:51.564122903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:00:51.566483435Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59CFCFD95EC9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:00:52.219748254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:00:52.221967043Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59CFF6EBDEA9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T03:00:54.597814058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:00:56.268647936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:00:56.271580018Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D0E848309C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=info ts=2026-07-02T03:00:57.238325364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:00:57.240818103Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D1220F5B17, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T03:00:59.597889358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:01:04.597850464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:01:04.801598036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:01:04.803911147Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D2E4DBA48D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:01:09.598149665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:01:09.788036753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:01:09.790341613Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D40E124C36, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T03:01:11.939443179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:01:11.941812548Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D48E4FF6A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:01:12.169526531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:01:12.174026097Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D49C2787EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:01:12.174045838Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:01:12.174070221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:01:12.182680327Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D49CAAF149, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:01:14.480244864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:01:14.48497424Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D525E541F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:01:14.597700312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:01:14.597711104Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:01:14.597740916Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:01:17.476379536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T03:01:17.478779921Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D5D8572A3D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T03:01:18.211882178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:01:18.214180425Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D6042C88AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:01:18.485052266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:01:18.487382952Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D614769844, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:01:18.631880487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:01:18.634083121Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D61D343CD3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:01:18.634098627Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:01:18.634116493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:01:18.635983349Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D61D52EAA4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:01:19.236456294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T03:01:19.238743022Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D6413E4090, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-07-02T03:01:19.238764219Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:01:19.238785599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="25 kB" avg_uncomp="25 kB" idle=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:01:19.240681204Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D6415DA041, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-07-02T03:01:19.598084123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:01:23.506915495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T03:01:23.509178741Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D73FC8FBB5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T03:01:24.597320701Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:01:25.354303751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:01:25.356689269Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D7ADE6DDD9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:01:27.018361923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T03:01:27.020900798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:01:27.020993672Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D81117E1EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:01:27.021009317Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:01:27.021026827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:01:27.02282051Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D81138A918, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:01:27.025110166Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D8115A66BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:01:27.114316068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:01:27.12067562Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D8170B2F0C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:01:29.48134088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:01:29.483804346Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D8A3E51653, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:01:29.483821816Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:01:29.483841084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:01:29.485668544Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D8A40350E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-02T03:01:29.597600832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:01:30.232839807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:01:30.235102188Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D8D0AE2C16, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:01:30.268154569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:01:30.274703846Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D8D30AC75B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:01:32.128579336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:01:32.133306945Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D941D1AE2B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T03:01:34.597564328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:01:35.129515505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:01:35.131903989Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59D9F48C7894, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T03:01:35.603307108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:01:35.60587647Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59DA10CA7857, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=info ts=2026-07-02T03:01:39.196076276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:01:39.198498661Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59DAE6EF7EDD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:01:39.198515068Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:01:39.198533741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:01:39.200673058Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59DAE7118340, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T03:01:39.597917822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:01:40.606337511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:01:40.609044851Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59DB3AFF0C12, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:01:43.036611421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:01:43.039169333Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59DBCBDB6829, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T03:01:44.57912883Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:01:44.596943706Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:01:44.596972298Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:01:44.597980225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:01:45.628598851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:01:45.631085338Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59DC66579A68, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T03:01:49.597320339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:01:52.145073172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:01:52.147532949Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59DDEAC200E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T03:01:54.59731497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:01:57.067851873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="314 B" avg_comp="314 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:01:57.070239533Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59DF102D8820, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:01:57.070255905Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:01:57.07027615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:01:57.072353538Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59DF104D42FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T03:01:57.534227931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T03:01:57.536535884Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59DF2BF7A893, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-07-02T03:01:57.536553893Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:01:57.536575575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="324 B" avg_comp="324 B" total_uncomp="112 kB" avg_uncomp="112 kB" idle=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:01:57.538624793Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59DF2C1A34FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T03:01:59.310730298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:01:59.315193382Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59DF95FD0AEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:01:59.378443839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:01:59.384311545Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59DF9A1BEE4F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:01:59.384332239Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:01:59.38435448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:01:59.386100432Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59DF9A39449D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:01:59.597442913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:02:03.443377578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:02:03.445657831Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E08C2EFA59, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T03:02:04.598232207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:02:06.532032977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:02:06.534469284Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E144486439, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:02:06.534485091Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:02:06.534504668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:02:06.536349743Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E14468E65F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:02:07.897451157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:02:07.899815199Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E195AB992B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T03:02:08.359986807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:02:08.362268179Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E1B13C7BBD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:02:09.110867066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:02:09.113169332Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E1DDFE3BAB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T03:02:09.313674105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:02:09.315936911Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E1EA14644D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:02:09.315952788Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:02:09.31597224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:02:09.317730682Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E1EA3166C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:02:09.598155231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:02:11.015364678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T03:02:11.017669977Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E24F81B763, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T03:02:11.017686292Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:02:11.017706005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="182 B" avg_comp="182 B" total_uncomp="17 kB" avg_uncomp="17 kB" idle=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:02:11.019472914Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E24FA03725, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:02:12.185841902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:02:12.188290901Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E29546EF7F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T03:02:12.460864064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:02:12.46352337Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E2A5ADB669, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:02:12.500693533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="920 B" avg_comp="920 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:02:12.503008616Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E2A80AA8B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:02:12.503023452Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:02:12.503042728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:02:12.504868205Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E2A82A0328, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:02:12.809887168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:02:12.812324815Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E2BA797058, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:02:14.59729648Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:02:14.597302629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:02:14.597329164Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:02:14.604379883Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-07-02T03:02:14.606049467Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008616202s
level=info ts=2026-07-02T03:02:15.763613005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:02:15.765934414Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E36A8644A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:02:16.483077278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:02:16.485371334Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E395689B4A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:02:18.407757621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:02:18.410236856Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E4082435FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T03:02:19.597450715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:02:19.780698616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:02:19.782920279Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E459F58C75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:02:20.467217236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:02:20.469587933Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E482E2EDAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:02:21.225447769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:02:21.227762021Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E4B0131151, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:02:21.32898511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:02:21.3312126Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E4B63EA4BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:02:21.331232232Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:02:21.331253916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:02:21.333172256Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E4B65D73A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:02:24.393831563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:02:24.400768038Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E56D3239BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T03:02:24.597524192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:02:25.158430554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:02:25.164852591Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E59ABE80F8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:02:25.704546428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:02:25.706800088Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E5BB0C2A71, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:02:29.598303741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:02:29.887441213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:02:29.889799929Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E6B45F52A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T03:02:30.583958829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:02:30.586232644Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E6DDE34D31, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T03:02:32.519622209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:02:32.522448328Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E751477ABD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=info ts=2026-07-02T03:02:34.401889228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:02:34.404266937Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E7C17587CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:02:34.598106444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:02:39.59750026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:02:39.72884518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:02:39.731361897Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E8FEF87586, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:02:41.630972636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:02:41.633269874Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59E97057355A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:02:44.579022026Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:02:44.597939863Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:02:44.597940848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:02:44.59795696Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T03:02:44.597969989Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:02:44.598091423Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-07-02T03:02:44.598099444Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T03:02:44.598117298Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.015µs success=true
level=info ts=2026-07-02T03:02:46.274587901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:02:46.279215232Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59EA85422769, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:02:47.57289831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:02:47.578219131Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59EAD2B0C2DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:02:49.59790673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:02:51.917984037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:02:51.92043857Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59EBD580746E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:02:53.542704004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:02:53.545185402Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59EC36570391, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-07-02T03:02:54.597286707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:02:56.07654263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:02:56.079283328Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59ECCD62B2B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:02:57.562350535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:02:57.564792184Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59ED25EDD873, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:02:59.293264252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:02:59.295924932Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59ED8D1C5C93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T03:02:59.597289008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:02:59.979438958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:02:59.981835572Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59EDB5FED95E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:03:00.739519013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:03:00.741853052Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59EDE34D28E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:03:00.857241012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:03:00.859601355Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59EDEA514802, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T03:03:03.036897564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:03:03.039337029Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59EE6C3D4041, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:03:03.300001916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:03:03.302439255Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59EE7BEC2115, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:03:03.839907314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:03:03.842270058Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59EE9C1976EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:03:04.597736145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:03:09.598075215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:03:13.570718958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:03:13.57320097Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59F0E01AAD93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T03:03:14.59721835Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:03:14.597256416Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:03:14.597300191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:03:17.063216213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:03:17.070204659Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59F1B08C06CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:03:17.618848681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:03:17.625116694Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59F1D19F7AD5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:03:18.600011735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:03:18.602372651Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59F20BDE57D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:03:19.598178563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:03:20.648722432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:03:20.651080753Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59F285FC1B6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T03:03:22.047779312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:03:22.050426006Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59F2D961BED1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=warn ts=2026-07-02T03:03:24.598065578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:03:26.837063706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:03:26.841667245Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59F3F6F7C8EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T03:03:29.597662938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:03:34.597841078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:03:37.835248725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:03:37.84113806Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59F68696A090, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:03:39.096526647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:03:39.09896862Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59F6D18F7D19, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T03:03:39.597838938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:03:40.257856595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:03:40.260248161Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59F716C777D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:03:40.500207336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:03:40.502746102Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59F7253AC736, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:03:42.337968584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:03:42.340244623Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59F792C20EA4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T03:03:44.579135098Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T03:03:44.597444768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:03:44.59743861Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:03:44.597474631Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:03:47.177568142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:03:47.17994668Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59F8B3393818, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:03:48.581058565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:03:48.58337925Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59F906E11F45, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T03:03:49.59779111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:03:50.729234151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:03:50.731640319Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59F986EBB07B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:03:51.095666788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:03:51.098073483Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59F99CC2D26B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:03:51.587576822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:03:51.589900563Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59F9BA14382A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:03:53.197755336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:03:53.200176542Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59FA1A0E91B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T03:03:54.065521226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:03:54.067875286Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59FA4DC6D6D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:03:54.598255122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:03:57.539251853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:03:57.541767922Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59FB1CD65DAE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:03:59.57629491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:03:59.578544576Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59FB963E1925, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:03:59.597783765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:04:04.598242771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:04:05.334905413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:04:05.337299308Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59FCED7D2A89, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:04:07.389449722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:04:07.396170212Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59FD6835BE5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:04:08.22890755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:04:08.23373024Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59FD9A205D25, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T03:04:09.598128802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:04:10.63605047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:04:10.641183722Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59FE29A054EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:04:11.301066185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:04:11.306252024Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE59FE5143822F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T03:04:14.596997796Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:04:14.597028803Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:04:14.598085342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:04:19.598229611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:04:20.116260256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:04:20.118902017Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A005E88761C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=info ts=2026-07-02T03:04:20.88790552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:04:20.890253676Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A008C844982, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:04:23.415456236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:04:23.417818863Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A01232B312E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T03:04:23.703519435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:04:23.705888222Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A013457571D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:04:24.59761675Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:04:29.563145129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="513 B" avg_comp="513 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:04:29.565631608Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A02919C2D1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T03:04:29.597831576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:04:33.0630811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:04:33.065448421Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A036236DDA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T03:04:34.597902499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:04:34.900610678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:04:34.903119537Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A03CFBE2D5B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:04:35.290161505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:04:35.292667664Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A03E6F6EA80, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:04:36.429831877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:04:36.432342008Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A042AE4FA4C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T03:04:36.606937416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:04:36.609295625Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A0435729F64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:04:38.030451331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:04:38.03292636Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A048A4B9909, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:04:39.096074844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:04:39.098469259Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A04C9CF5822, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-02T03:04:39.597937304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:04:44.579027401Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T03:04:44.597265935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:04:44.597278971Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:04:44.597303636Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:04:45.051933126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:04:45.054343038Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A062CCEEC67, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:04:47.306419895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:04:47.308866976Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A06B32F70B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:04:47.404218374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:04:47.406689083Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A06B904304C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:04:48.735203084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:04:48.737661814Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A070859F435, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T03:04:49.597998679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:04:50.085551028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:04:50.088088708Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A0758D618A6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:04:51.606593971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:04:51.608981435Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A07B37E5FC0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T03:04:54.597315973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:04:59.537092415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:04:59.544511332Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A098C7D3021, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T03:04:59.597643948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T03:05:00.17236418Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE5A09B1EBFF6F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T03:05:04.597716513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:05:05.927046265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:05:05.931946855Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A0B09343534, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=warn ts=2026-07-02T03:05:09.597893572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:05:10.075204613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:05:10.080031041Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A0C0075063B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:05:12.896448327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:05:12.901592963Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A0CA89F131E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:05:14.597689286Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:05:14.597717676Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:05:14.597713881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:05:17.257391604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:05:17.259928398Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A0DAC67236B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:05:19.59750739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:05:20.834517913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:05:20.836777886Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A0E819BE957, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:05:22.177580497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:05:22.179947441Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A0ED1A9E2E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T03:05:23.455518436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:05:23.457828714Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A0F1DD622FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T03:05:24.012717726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:05:24.015064465Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A0F3F0CE52E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T03:05:24.598038503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:05:24.816141997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:05:24.818461347Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A0F6EEECE45, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:05:26.961168571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:05:26.963528538Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A0FEEC9FD5D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:05:29.598023068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:05:30.97479714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:05:30.977172937Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A10DE0657C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:05:31.829531661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:05:31.831795576Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A1110F67DDC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:05:32.284047856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:05:32.286347947Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A112C0E0E81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:05:33.898948818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:05:33.901437099Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A118C51F48B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:05:34.289568373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:05:34.291899054Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A11A398D8DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:05:34.598065692Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:05:35.12473108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:05:35.127265205Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A11D561FD62, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:05:36.323296084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:05:36.325763752Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A121CD1D75C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:05:39.087667004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:05:39.090834028Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A12C19BE893, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T03:05:39.598032715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:05:41.590134101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:05:41.592691265Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A1356C12B45, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T03:05:44.57928651Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:05:44.597351313Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:05:44.597381479Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:05:44.597417067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:05:44.929397069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:05:44.937534382Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A141E1F609B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:05:46.408004589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:05:46.413944261Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A14761ED758, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=warn ts=2026-07-02T03:05:49.597600663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:05:53.382445464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:05:53.387290834Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A1615C5665E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T03:05:54.597355414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:05:59.597671841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:06:00.080273096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="663 B" avg_comp="663 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:06:00.085007007Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A17A4FC201D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:06:01.763108685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:06:01.765452548Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A180925498D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T03:06:04.597464126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:06:05.578318615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:06:05.580738082Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A18EC8E557A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T03:06:09.548714152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:06:09.551158026Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A19D935DEA3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:06:09.598031075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:06:12.629140988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:06:12.631532516Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A1A90D0AC14, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:06:13.471735899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:06:13.474102424Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A1AC3092B04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:06:13.4741174Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:06:13.474136875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:06:13.47593052Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A1AC32829A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:06:14.597834089Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:06:14.597848331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:06:14.597864331Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:06:16.774954533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:06:16.777425585Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A1B87EE0205, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:06:17.017040034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:06:17.019364977Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A1B9659E80B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:06:17.85661786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:06:17.858959306Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A1BC865EACE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:06:19.396449512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:06:19.398733261Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A1C242CEBE2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T03:06:19.597935815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:06:21.06378817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:06:21.066062308Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A1C878EFC00, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:06:21.711038454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:06:21.713419952Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A1CAE22F8D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:06:21.922966028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:06:21.925375972Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A1CBAC5A5D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T03:06:24.598214865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:06:25.790012633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:06:25.796953673Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A1DA189367C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:06:25.796972958Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:06:25.796997659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="64 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:06:25.80744282Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A1DA22B2E69, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:06:26.784059173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:06:26.788966549Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A1DDCA9CBF6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T03:06:27.23804348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:06:27.240309092Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A1DF7916A3A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:06:27.809228433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="314 B" avg_comp="314 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:06:27.811633962Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A1E199E1F72, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T03:06:29.598013236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:06:31.390430946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:06:31.392802824Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A1EEF12C473, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:06:33.080730588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:06:33.083126794Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A1F53D3503C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T03:06:34.597962032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:06:39.597943391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:06:40.364608391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:06:40.367062278Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2105FA9549, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:06:40.54662308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:06:40.549273507Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2110D6B013, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=info ts=2026-07-02T03:06:40.874397144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:06:40.881223982Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2124A22245, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:06:43.679014021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:06:43.68668861Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A21CBD894A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:06:44.307324569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:06:44.312434424Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A21F12433C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:06:44.312454509Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:06:44.31247609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:06:44.314288995Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A21F143CE86, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:06:44.578313146Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:06:44.597849371Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:06:44.597878751Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:06:44.5978772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:06:48.347082319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:06:48.349436864Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A22E1C50F1E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-07-02T03:06:49.598082155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:06:54.597574587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:06:55.615511655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:06:55.618058518Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2493022342, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:06:56.131279365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:06:56.133746557Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A24B1BEDEA6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:06:59.597926874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:07:00.679335981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:07:00.681782627Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A25C0D4F1E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:07:00.681803086Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:07:00.681822096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:07:00.683626853Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A25C0F49564, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:07:02.877460778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:07:02.879807051Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2643D7E73F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:07:02.951892989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:07:02.954202597Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2648482C6E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:07:03.385549327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:07:03.387840698Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2662210173, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:07:04.597259897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:07:07.42360102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:07:07.42604771Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2752D245C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:07:09.123231633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:07:09.125510943Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A27B81EF4A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:07:09.597882321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:07:11.560304669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:07:11.562920648Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2849653455, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T03:07:11.89068759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="920 B" avg_comp="920 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:07:11.893134169Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A285D14E293, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:07:13.7409723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:07:13.743364172Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A28CB5D34E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-07-02T03:07:13.743380385Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:07:13.743398801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:07:13.745269029Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A28CB7C84B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:07:14.031172213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:07:14.033542639Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A28DCA94E80, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:07:14.230072013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:07:14.232550321Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A28E8860C7D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T03:07:14.596951967Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:07:14.596982958Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:07:14.598017798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:07:14.604330613Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-07-02T03:07:14.606228099Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007998464s
level=info ts=2026-07-02T03:07:16.453699469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:07:16.456119892Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A296D0EE936, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T03:07:16.507251251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:07:16.514510538Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2970888600, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:07:17.104284016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:07:17.110377466Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A29940DD8E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:07:19.598106694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:07:20.072378545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:07:20.07476296Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2A44BF6EEE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T03:07:24.598195272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:07:26.224356323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:07:26.226751121Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2BB36E91BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:07:29.297063269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:07:29.299525404Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2C6A943A00, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-07-02T03:07:29.299543385Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:07:29.299561434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:07:29.301371764Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2C6AB37C6E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-07-02T03:07:29.597532558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:07:32.036446959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:07:32.03912663Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2D0DDDAB2B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:07:32.039142621Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:07:32.039162561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:07:32.041397069Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2D0E01C5F8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=info ts=2026-07-02T03:07:32.645001627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:07:32.647342755Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2D3221446C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:07:32.86187704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:07:32.864260348Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2D3F0FA483, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:07:33.373691081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:07:33.378261179Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2D5DB1E739, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:07:33.378278712Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:07:33.378298117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:07:33.386367987Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2D5E2F380E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T03:07:34.597306929Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:07:37.896989835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:07:37.901789577Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2E6B5165F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T03:07:39.598021879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:07:41.144702392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:07:41.147170914Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2F2CC2A043, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T03:07:43.714195477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:07:43.716806715Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2FC5E9C7BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=info ts=2026-07-02T03:07:44.579098819Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:07:44.597881569Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:07:44.597907777Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:07:44.597908771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:07:44.597914801Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T03:07:44.598035149Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-07-02T03:07:44.598041927Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T03:07:44.598060239Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.862µs success=true
level=info ts=2026-07-02T03:07:44.622068617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:07:44.626777568Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A2FFC28C45F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:07:45.102390343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:07:45.10730542Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3018CDCF3D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:07:45.107326499Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:07:45.10734642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:07:45.109174659Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3018EC0D49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:07:47.657028178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:07:47.65939815Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A30B0EB29CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:07:49.351650847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:07:49.354064832Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3115EC4602, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:07:49.598130771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:07:51.442443813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:07:51.444755629Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A31928B3220, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:07:51.44477366Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:07:51.444795114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:07:51.446729292Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3192AA91AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T03:07:54.598271044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:07:56.991059371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:07:56.993278733Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A32DD438B84, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:07:57.816447475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:07:57.818802632Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A330E765C22, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:07:57.818825551Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:07:57.818852403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:07:57.820974653Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A330E98EE6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T03:07:59.039112489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:07:59.041781437Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A33575A60B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T03:07:59.597612169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:08:01.074256992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:08:01.081317836Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A33D0ED0C41, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:08:04.278534193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:08:04.284972202Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A348FDFB830, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:08:04.284992864Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:08:04.285016376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:08:04.286841667Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A349000A369, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:08:04.597522507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:08:04.766893021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:08:04.769205861Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A34ACBE06C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:08:05.013011259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:08:05.015548245Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A34BB6BAC76, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T03:08:09.597446834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:08:09.695703629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:08:09.698078658Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A35D286B8AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:08:10.316313735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:08:10.318919536Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A35F784E2BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:08:11.110452208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:08:11.112763388Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3626D8FB5E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:08:11.112777738Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:08:11.11279792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:08:11.114594146Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3626F7857C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:08:11.339178461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:08:11.341491327Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A36347B13D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:08:11.933339554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:08:11.935667586Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3657E52632, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T03:08:12.024099162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:08:12.026421799Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A365D4DCA5D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:08:12.026443382Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:08:12.026465092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:08:12.028264937Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A365D6D4844, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T03:08:14.333877204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:08:14.33652318Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A36E6FD15C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=info ts=2026-07-02T03:08:14.356547585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:08:14.358682248Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A36E8532691, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:08:14.358694507Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:08:14.358711281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:08:14.360727377Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A36E871D27A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:08:14.597875305Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:08:14.597878795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:08:14.597906325Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:08:14.811831132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:08:14.814237705Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A370378CBC3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:08:14.814252999Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:08:14.814273202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:08:14.816048337Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A370397F66F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T03:08:15.170440822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:08:15.175128111Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3718FBC5E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:08:17.73753678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:08:17.744063572Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A37B21A6371, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T03:08:19.560535756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:08:19.562902999Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A381E83CA03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T03:08:19.597932754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:08:23.307181479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:08:23.309530563Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A38FDD5D7CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:08:23.328591916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:08:23.330767403Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A38FF1A4A0E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T03:08:24.598185946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:08:25.432296496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:08:25.434525373Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A397C7E8CEB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:08:25.933968817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:08:25.936353947Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A399A66E7AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T03:08:28.725418951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:08:28.727735025Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3A40C8569F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-02T03:08:29.598260634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:08:29.877205097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:08:29.879625043Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3A85700FC6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:08:31.923147959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:08:31.925507199Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3AFF61C94E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:08:34.597319121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:08:34.599379095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:08:34.601644788Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3B9EE46A9F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:08:34.601657951Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:08:34.601676018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:08:34.603464348Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3B9F02F8F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:08:39.598044794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:08:44.579004817Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:08:44.597515074Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:08:44.597532991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:08:44.597544292Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:08:45.129963652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:08:45.133394979Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3E12A2FEB7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:08:45.230730103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:08:45.233051581Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3E1893D0E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T03:08:45.346290994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:08:45.348632526Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3E1F77511F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:08:45.348648705Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:08:45.348668061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:08:45.350419951Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3E1F95453D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:08:46.159649738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:08:46.16190228Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3E4FF0616D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:08:47.584434354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:08:47.586704678Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3EA4DD0769, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:08:47.586719994Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:08:47.58673941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:08:47.58849736Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3EA4FB08E7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:08:47.610573144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:08:47.968742652Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3EBBA23C94, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:08:49.137718692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:08:49.142581272Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3F01993706, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T03:08:49.597603141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:08:51.194463941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:08:51.196803831Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A3F7C0A0BD0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T03:08:54.598292734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:08:55.496617476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:08:55.499201211Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A407C7972FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:08:56.100615352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:08:56.105187137Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A40A09A782E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:08:57.38772613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:08:57.392509047Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A40ED551415, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:08:58.068381594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:08:58.071032011Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A4115C501C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=info ts=2026-07-02T03:08:58.938754506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:08:58.941184857Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A4149A4B728, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:08:58.941202333Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:08:58.941220581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:08:58.943000664Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A4149C3477C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:08:58.954012115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:08:58.955844083Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A414A866CFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-07-02T03:08:59.598311989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:09:00.825131911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:09:00.827437662Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A41BA131C29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:09:00.926599496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:09:00.928964666Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A41C01E9018, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:09:01.017539901Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE5A41C5684CC1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T03:09:02.424032577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:09:02.426373568Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A4219613612, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:09:04.251459023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:09:04.253709878Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A42864BD4F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-02T03:09:04.597550866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:09:06.396886296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:09:06.399359734Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A43062F3809, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T03:09:09.598297344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:09:10.682061217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:09:10.684509305Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A440599C5FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:09:13.888719853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:09:13.891182987Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A44C4BADBEE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:09:14.596996345Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:09:14.597028482Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:09:14.598099216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:09:15.875191263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:09:15.877481254Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A453B205D62, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:09:19.597628505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:09:20.781092159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:09:20.783442228Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A465F8AB4A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:09:23.419816447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:09:23.422224535Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A46FCD43704, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T03:09:24.597569763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:09:27.862608421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:09:27.864980953Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A4805A2CE30, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:09:29.598143489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:09:31.953754884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:09:31.956170114Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A48F97D6960, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T03:09:34.597517289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:09:39.539454547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:09:39.541886248Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A4ABDA192DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T03:09:39.597435344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:09:40.935926859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:09:40.938328342Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A4B10DED904, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:09:41.503015119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:09:41.505468656Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A4B32ABFCD2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:09:42.610045003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:09:42.61253838Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A4B74A805D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T03:09:44.578507111Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:09:44.59711858Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:09:44.597149456Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:09:44.598221473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:09:45.176718553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:09:45.1791365Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A4C0DA37B7C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T03:09:45.536920721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="513 B" avg_comp="513 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:09:45.539241961Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A4C231AF174, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:09:47.535655378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:09:47.54251303Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A4C9A828163, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T03:09:49.597643756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:09:51.690193042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:09:51.695282293Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A4D9206B452, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=info ts=2026-07-02T03:09:51.859700041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:09:51.864051956Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A4D9C18AC01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:09:51.932253977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:09:51.937138879Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A4DA0722F2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:09:54.503829847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:09:54.506120994Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A4E39936390, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-02T03:09:54.597778587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:09:55.27814938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:09:55.280473549Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A4E67BAC372, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:09:58.133619581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:09:58.136054519Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A4F11ECED0A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T03:09:58.835349811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:09:58.83777124Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A4F3BC195C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:09:59.597731812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:10:01.150759881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:10:01.153082659Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A4FC5C3A0E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T03:10:04.597384934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:10:06.141631787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:10:06.14464093Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A50EF3EB5D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:10:09.025605025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:10:09.028216652Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A519B25E3E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:10:09.597867697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:10:14.597047378Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:10:14.597079462Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:10:14.598157177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:10:15.373709805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:10:15.376132206Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A531585D008, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T03:10:19.598228735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:10:20.85911993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:10:20.86150244Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A545C79A78B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T03:10:23.195577837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:10:23.19786213Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A54E7BBDCFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:10:23.409428779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:10:23.411714463Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A54F47A91C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:10:24.234035304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:10:24.236335651Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A5525A1E034, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T03:10:24.597931406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:10:28.330683231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:10:28.333066329Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A5619D05943, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:10:28.441452158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:10:28.444121938Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A56206DE0FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T03:10:29.597640329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:10:31.85222477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:10:31.859317221Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A56EBFCDC8E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T03:10:34.597703799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:10:34.722043871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:10:34.729441198Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A57970FE2E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:10:37.876750853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:10:37.881511181Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A5852F232CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T03:10:39.59783957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:10:39.691170863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="663 B" avg_comp="663 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:10:39.693794542Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A58BEF4FFA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:10:40.584713653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:10:40.586978288Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A58F4349541, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:10:42.552464743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:10:42.554885883Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A59697F0C72, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T03:10:44.578531701Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:10:44.596951358Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:10:44.59697486Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:10:44.59801957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:10:44.865946983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:10:44.868257671Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A59F3639E23, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T03:10:46.832458887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:10:46.834946657Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A5A689C1697, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:10:48.284953874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:10:48.287299684Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A5ABF2E23DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-07-02T03:10:49.597957964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:10:50.045969091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:10:50.048619106Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A5B2827CC7E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=info ts=2026-07-02T03:10:52.705371239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:10:52.707739185Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A5BC6A8C007, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T03:10:54.59784817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:10:56.091691992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:10:56.093962989Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A5C907F7036, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:10:57.208913914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:10:57.211197354Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A5CD3162D2D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:10:59.597669411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:11:03.52051123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:11:03.522839343Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A5E4B49F97D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:11:04.428603213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:11:04.431009827Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A5E816C03E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T03:11:04.59732985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:11:04.995879077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:11:04.998311863Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A5EA33C6D44, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T03:11:09.597806504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:11:10.557261814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:11:10.559878347Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A5FEEB91FFF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:11:14.597627448Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:11:14.597659168Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:11:14.597680195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:11:15.908651259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:11:15.910943871Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A612DADC10C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:11:18.706090466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:11:18.712902026Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A61D4AFBBAE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T03:11:19.598231851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:11:21.8009544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:11:21.807939254Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A628D28CFA0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:11:23.599515864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:11:23.604007691Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A62F8373390, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:11:24.043074466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:11:24.04802263Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A6312AFC12F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T03:11:24.597446521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:11:26.371858019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:11:26.374390059Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A639D56AB82, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:11:27.24637969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="314 B" avg_comp="314 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:11:27.248722515Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A63D177100F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T03:11:29.59768653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:11:31.094701456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:11:31.097173027Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A64B6D873E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-02T03:11:34.597243327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:11:35.554651651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:11:35.557077187Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A65C0AD20DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:11:36.573662426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:11:36.576017106Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A65FD68D355, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T03:11:38.234601291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:11:38.236904898Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A666068F27B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:11:38.351126854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:11:38.353426719Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A66675A3566, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-07-02T03:11:39.597423282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:11:40.797813403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:11:40.800384561Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A66F9327155, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=info ts=2026-07-02T03:11:44.578639657Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:11:44.597203077Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:11:44.597231119Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:11:44.597257647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:11:46.421196534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:11:46.423804827Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A6848610AC1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T03:11:49.597454696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:11:51.500082977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:11:51.502780341Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A69771C5978, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:11:51.805186478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:11:51.807618884Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A6989479AD3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:11:53.763760278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:11:53.766359432Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A69FE082804, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:11:54.59780617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:11:54.694145013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:11:54.696546794Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A6A357A50DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:11:56.665180971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:11:56.667655271Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A6AAAF6E46D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T03:11:56.876294009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:11:56.878630846Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A6AB78A171A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:11:59.167970328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:11:59.174864014Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A6B4068D776, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T03:11:59.598084304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:12:01.969084907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:12:01.975162125Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A6BE750F3D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T03:12:04.5976805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:12:09.598053836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:12:10.552333786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:12:10.5547155Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A6DE6B35432, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:12:14.597969459Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:12:14.597973844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:12:14.597998579Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:12:14.602137542Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-07-02T03:12:14.603989167Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.005813248s
level=info ts=2026-07-02T03:12:15.466631434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:12:15.469143469Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A6F0B9FEDAB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T03:12:15.548252849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="920 B" avg_comp="920 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:12:15.55078725Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A6F107C2703, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:12:18.472664914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:12:18.475151312Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A6FBECBBE9D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:12:18.610720288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:12:18.613038973Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A6FC7032AC7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:12:18.613053969Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:12:18.61307212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:12:18.614813247Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A6FC721694A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T03:12:19.597437744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:12:22.060327713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:12:22.06483018Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A7094C3A0BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T03:12:24.597280251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:12:26.616980945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:12:26.622840055Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A71A46F75FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T03:12:27.20045851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:12:27.202776006Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A71C70037F0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:12:28.092279002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:12:28.094655773Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A71FC28B021, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T03:12:29.597818668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:12:31.100066084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:12:31.102828981Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A72AF72C2E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=info ts=2026-07-02T03:12:32.959518188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:12:32.961905913Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A731E45F861, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:12:34.225610929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:12:34.227873525Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A7369BAEE9E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T03:12:34.59825276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:12:34.981075496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:12:34.983347873Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A7396C31F69, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:12:35.93373637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:12:35.936064411Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A73CF8B5061, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:12:39.025296338Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE5A7487AED3D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T03:12:39.598015734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:12:39.719344939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:12:39.721757021Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A74B1305264, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T03:12:41.825108056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:12:41.832117647Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A752EFA775B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:12:41.832135847Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:12:41.832159872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:12:41.842091702Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A752F9400B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:12:44.067391916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:12:44.072093272Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A75B47E5BEE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:12:44.57899437Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:12:44.59698945Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T03:12:44.596999827Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:12:44.597021801Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:12:44.597115255Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-07-02T03:12:44.597123201Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T03:12:44.597140807Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.858µs success=true
level=warn ts=2026-07-02T03:12:44.597338971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:12:45.383595389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:12:45.385985971Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A7602CF1B25, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:12:46.909009632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:12:46.911385434Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A765DBA209C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:12:46.949436083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:12:46.951567822Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A7660207A50, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T03:12:49.59802861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:12:50.347046297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:12:50.349343126Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A772AA5A989, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:12:53.938776434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:12:53.946785496Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A780111D558, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T03:12:54.598299697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:12:57.066730253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:12:57.072678442Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A78BB61EAE2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T03:12:59.597989799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:13:00.398600971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:13:00.400832481Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A7981C317FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:13:01.518610748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:13:01.520905541Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A79C48592E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:13:04.598232628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:13:05.179340247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:13:05.181852129Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A7A9EB9B498, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:13:05.181868757Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:13:05.181886957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:13:05.183594479Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A7A9ED90AD4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T03:13:09.59739337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:13:10.716004357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:13:10.718275217Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A7BE8BAB879, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T03:13:11.534452198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:13:11.536670094Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A7C19826CDD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:13:14.596996207Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:13:14.597026385Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:13:14.598102993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:13:14.765635671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:13:14.770574162Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A7CDA4298A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:13:14.904683531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:13:14.911091397Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A7CE2A3BFE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:13:18.224811691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:13:18.227310401Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A7DA84CA112, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:13:19.597457582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:13:20.672074444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:13:20.674442415Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A7E3A29722E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:13:22.116056382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:13:22.119656819Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A7E904DA9D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:13:23.841659993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:13:23.844278199Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A7EF716BFDB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=warn ts=2026-07-02T03:13:24.597598396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:13:25.331536885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:13:25.333873739Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A7F4FE23BA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T03:13:27.681048719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:13:27.683195268Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A7FDBEC2EF1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T03:13:29.597758291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:13:30.023255302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:13:30.025883273Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A80678A9D82, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:13:31.719554205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:13:31.721819881Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A80CCA3B897, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:13:32.155210863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:13:32.157491375Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A80E69B05D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:13:33.930458255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T03:13:34.598234146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:13:35.084249769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:13:35.090199571Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A81956855ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:13:35.090217482Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:13:35.090238049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:13:35.092013986Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A819586F7DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:13:35.321318975Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A81A32F176F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:13:35.912010533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:13:35.916828902Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A81C6AC6D11, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:13:39.597672116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:13:39.861554887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:13:39.864012034Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A82B1F2881E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T03:13:40.733120913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:13:40.735418439Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A82E5E47A1F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:13:44.578617356Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T03:13:44.597715412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:13:44.597738525Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:13:44.597762549Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:13:44.892672904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:13:44.8949366Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A83DDD0ACE5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T03:13:49.597887252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:13:50.078791043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:13:50.081170303Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8512F0EBF3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:13:53.967449723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:13:53.96974048Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A85FAB7C564, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T03:13:54.598236031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:13:55.134598109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:13:55.136709326Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8640470757, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:13:55.198217311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:13:55.200356587Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8644124EDD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:13:56.023600631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:13:56.025899956Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A867545E766, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:13:56.292600585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:13:56.294923861Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A86854EA9FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-07-02T03:13:56.294939456Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:13:56.294958218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:13:56.296726183Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A86856D0DA7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T03:13:57.432029553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:13:57.436531204Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A86C959E175, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:13:57.436550819Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:13:57.436573514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:13:57.444211036Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A86C9D184B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T03:13:59.598158588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:14:04.012601708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:14:04.017481829Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A88519B0639, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:14:04.597294608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:14:09.597294245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:14:09.830983253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:14:09.833424696Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A89AC431353, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T03:14:11.966896489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:14:11.969291848Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8A2B92B275, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:14:12.541708759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:14:12.544319039Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8A4DD5E71F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T03:14:14.597494243Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:14:14.59752606Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:14:14.5975468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:14:15.205824774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:14:15.210289627Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8AECC0773A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:14:15.69970449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:14:15.705712592Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8B0A425EE7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:14:18.828327104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:14:18.830749871Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8BC48BC88E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:14:18.830766805Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:14:18.830784101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:14:18.83279225Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8BC4ACB191, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:14:19.277354299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:14:19.27979117Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8BDF4F85CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:14:19.597843753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:14:22.731360495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:14:22.733734289Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8CAD2F8E1D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:14:22.733753107Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:14:22.733772051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:14:22.735519431Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8CAD4D2AC3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T03:14:23.358173889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:14:23.360477605Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8CD28AB2B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:14:23.360492904Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:14:23.36051134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:14:23.362236004Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8CD2A86E07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T03:14:23.637232221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:14:23.6399117Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8CE32F9B36, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:14:23.639928434Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:14:23.661123321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:14:23.663638134Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8CE49A17CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=warn ts=2026-07-02T03:14:24.598255821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:14:25.131117672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:14:25.133496682Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8D3C3787EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:14:29.59747757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:14:29.682718294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="513 B" avg_comp="513 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:14:29.685145536Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8E4B857F44, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:14:30.05511056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:14:30.057478401Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8E61B70077, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:14:31.455456325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:14:31.457851792Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8EB52DE999, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-07-02T03:14:31.457868511Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:14:31.457887906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:14:31.459646134Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8EB54CC73C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:14:32.386000564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:14:33.684246374Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8F39E2680F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T03:14:34.597978465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:14:34.775957198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:14:34.780380937Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8F7B38A14A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T03:14:35.93675197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:14:35.939001406Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8FC047BE04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:14:35.939015404Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:14:35.939033814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:14:35.941024132Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A8FC0684B44, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:14:37.46114515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:14:37.463904108Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A901B28BA6D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T03:14:37.53209724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:14:37.53468422Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A901F615670, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=info ts=2026-07-02T03:14:38.041306291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:14:38.043651908Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A903DB9E3C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:14:38.043666811Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:14:38.043686008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:14:38.045386336Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A903DD793C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-02T03:14:39.597585901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:14:43.502740283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:14:43.507159957Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A918360E75B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:14:43.731689297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:14:43.736373889Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A91910A5385, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:14:43.931931281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:14:43.934259318Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A919CD6A67E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:14:43.934275663Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:14:43.934294545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:14:43.936103502Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A919CF4D4F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:14:44.578462749Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:14:44.59773997Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:14:44.597765867Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:14:44.597788585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:14:47.639746757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:14:47.642046325Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A9279D6E649, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:14:48.493357788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:14:48.495753241Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A92ACB8926C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-07-02T03:14:49.597645282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:14:50.669838539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:14:50.672135297Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A932E7151B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:14:50.67215201Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:14:50.67217053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:14:50.673925643Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A932E8F253F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T03:14:53.621725536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:14:53.624156062Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A93DE64D4F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:14:54.444298721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:14:54.446659144Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A940F6B2596, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-02T03:14:54.598237382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:14:55.18773851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:14:55.190012974Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A943BBB2F1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:14:57.898460684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:14:57.900899807Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A94DD4EE287, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:14:58.330953119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:14:58.333381581Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A94F7164B03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:14:58.3333968Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:14:58.333430761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:14:58.33518385Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A94F7350A82, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:14:59.597451401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:15:00.895557953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:15:00.897897746Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A958FF197EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T03:15:01.376340479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:15:01.378658854Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A95AC99C137, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:15:01.378676278Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:15:01.378694758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:15:01.380479973Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A95ACB849D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:15:03.565591182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:15:03.568187539Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A962F187259, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=warn ts=2026-07-02T03:15:04.597797967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:15:06.499730127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:15:06.502091524Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A96DDFBFBCD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T03:15:08.559183134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:15:08.561452503Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A9758BB165F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:15:08.760273953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:15:08.762748568Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A9764B93419, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:15:08.762763945Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:15:08.762783154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:15:08.764595879Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A9764D8561F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:15:09.597760786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:15:09.688169441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:15:09.690441866Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A979C05A492, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:15:10.34230837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:15:10.344617905Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A97C3037DF1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:15:11.269568072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:15:11.273713345Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A97FA64A6A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:15:12.002878854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:15:12.008635183Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A982632EA65, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:15:12.278363375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:15:12.280587112Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A983667FCA6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:15:12.280602606Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:15:12.280620481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:15:12.282286004Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A98368558BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:15:14.363504871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:15:14.365930717Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A98B2B42937, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T03:15:14.597333018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:15:14.597358435Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:15:14.59738875Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:15:15.047441217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:15:15.049758013Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A98DB762376, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:15:15.088930353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:15:15.0910016Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A98DDED59AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T03:15:19.5975066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:15:20.481192806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:15:20.483656382Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A9A1F58FF2A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:15:21.350010617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:15:21.352390424Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A9A532092BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:15:22.813043252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:15:22.815559011Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A9AAA57218C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:15:24.233456376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:15:24.241200787Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A9AFF4F1646, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T03:15:24.597307098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:15:25.368257238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:15:25.376307346Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A9B42F934D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:15:27.076899385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:15:27.083892669Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A9BA8BEAB9F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:15:27.666638889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:15:27.671475788Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A9BCBC59BA6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T03:15:29.597264145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:15:30.072103389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:15:30.074550584Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A9C5B013CE5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:15:32.297640932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:15:32.299981061Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A9CDFA7AAD2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:15:34.598062089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:15:38.359330647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:15:38.361678218Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A9E48F56411, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T03:15:39.597813746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:15:40.877936028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:15:40.880293464Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A9EDF148903, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:15:41.495948463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:15:41.498254652Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A9F03EAB6FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:15:43.950226723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="663 B" avg_comp="663 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:15:43.952645182Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A9F963567A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:15:44.209452588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:15:44.211797229Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5A9FA5A6FF3D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T03:15:44.578924134Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T03:15:44.597285103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:15:44.597306562Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:15:44.59732911Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:15:46.707072856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:15:46.709525183Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AA03A8714BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:15:46.709541296Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:15:46.709561712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:15:46.711330812Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AA03AA6A6D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T03:15:49.598077555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:15:54.597826564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:15:57.827270873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:15:57.831691733Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AA2D175A6D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T03:15:59.129513065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:15:59.132148754Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AA31EF9125D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=warn ts=2026-07-02T03:15:59.597930408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:16:00.881515933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:16:00.883864283Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AA387634EA0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:16:01.502213583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:16:01.504523503Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AA3AC61C2ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:16:01.785527618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:16:01.787930571Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AA3BD4658A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:16:03.419739219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:16:03.423939135Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AA41EC9735D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:16:04.313431189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:16:04.319554891Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AA4542B00A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-02T03:16:04.597758528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:16:05.005139574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:16:05.007302026Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AA47D2AE3E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T03:16:06.427220774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:16:06.42947902Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AA4D1EF5676, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:16:08.342107824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:16:08.344371248Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AA54411CD6E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:16:09.598280384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:16:13.159772423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:16:13.162477811Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AA6633D2BD6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:16:13.449348735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:16:13.451711207Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AA6747D0283, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:16:14.597583414Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:16:14.597613722Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:16:14.597611335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:16:16.840874556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:16:16.845490608Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AA73EC634F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:16:17.180459826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:16:17.191490496Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AA75366577D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:16:17.25354078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:16:17.258083929Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AA7575E4937, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:16:19.59784448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:16:21.703714856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:16:21.706056766Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AA8607CDA61, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:16:24.597517685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:16:25.919208711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:16:25.921507407Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AA95BBFD6F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:16:28.499296388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:16:28.501732621Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AA9F58A02B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:16:29.597560817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:16:31.629768237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:16:31.632380016Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AAAB0243ADD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-07-02T03:16:34.598251906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:16:35.896722914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:16:35.898974575Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AABAE73A592, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:16:36.977011154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:16:36.979317144Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AABEED7D650, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T03:16:38.390814605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:16:38.393167801Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AAC431E5F26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T03:16:39.597498881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T03:16:42.724205616Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE5AAD45462D04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T03:16:43.009325661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:16:43.011626285Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AAD566604F0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:16:44.578460613Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:16:44.597587722Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:16:44.597612829Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:16:44.597609959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:16:45.258586236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:16:45.260986104Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AADDC789CD3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:16:47.509175566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=info ts=2026-07-02T03:16:47.51007574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:16:47.511954878Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AAE62A01ACA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:16:47.513135632Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AAE62B86C4F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T03:16:49.597655072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:16:50.29943333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:16:50.301661907Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AAF08EB9D2A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:16:52.370770578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="314 B" avg_comp="314 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:16:52.372991579Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AAF8461969A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T03:16:54.598233924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:16:57.603510142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:16:57.610176798Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AB0BC8A8EC2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:16:59.401445699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:16:59.403765342Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AB12771FFEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:16:59.598281826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:16:59.872052741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:16:59.876395179Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AB1439D929B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:17:00.306609157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:17:00.312574479Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AB15D9D4C2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:17:01.730155887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:17:01.732384672Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AB1B23EF9F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T03:17:04.598126115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:17:05.220017319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:17:05.222420146Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AB28243B012, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:17:06.458758121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:17:06.46111999Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AB2CC1836C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:17:06.746013956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:17:06.750429331Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AB2DD57B29E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T03:17:09.597713925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:17:10.672910829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:17:10.679537798Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AB3C788061D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:17:10.76261809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:17:10.764875766Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AB3CC9EB608, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:17:14.597981627Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:17:14.59797887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:17:14.598008111Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:17:14.60244572Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-07-02T03:17:14.604310934Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007059043s
level=warn ts=2026-07-02T03:17:19.597594027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:17:19.745939799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:17:19.74820804Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AB5E4124B61, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:17:23.760228313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:17:23.762563732Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AB6D35850BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T03:17:24.597715345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:17:26.94730491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:17:26.949692271Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AB7914F5080, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-07-02T03:17:29.598067133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:17:33.071380381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:17:33.073950499Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AB8FE557009, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=info ts=2026-07-02T03:17:34.255097878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:17:34.257366214Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AB944E1C355, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T03:17:34.597535703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:17:34.984717964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:17:34.986980887Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AB9705E495C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:17:37.039105876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="920 B" avg_comp="920 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:17:37.041474368Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AB9EAD35CFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:17:37.88471595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:17:37.887146898Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ABA1D3CB80C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:17:38.041453387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:17:38.043761875Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ABA2691D230, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T03:17:39.597434731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:17:43.720928062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:17:43.723189651Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ABB79175D20, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T03:17:43.7894533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:17:43.791736857Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ABB7D2C6436, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:17:44.578650955Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:17:44.59785921Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T03:17:44.597883844Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:17:44.597887576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:17:44.597909161Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:17:44.597976361Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-07-02T03:17:44.597984129Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T03:17:44.598002994Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.269µs success=true
level=info ts=2026-07-02T03:17:47.239689406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:17:47.242065726Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ABC4AD3E15A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T03:17:49.48643023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:17:49.490792731Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ABCD0DCFAFB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T03:17:49.597794611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:17:51.668211668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:17:51.960465639Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ABD64119118, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:17:53.870528445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:17:53.875270548Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ABDD63374E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:17:54.093879905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:17:54.098372457Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ABDE37F6CC1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T03:17:54.59742293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:17:56.065790467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:17:56.07030822Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ABE5908BDD4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:17:57.689721231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:17:57.692121636Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ABEB9B40C36, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:17:59.597268348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:17:59.878400434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:17:59.88084995Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ABF3C28C9AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:18:00.219001108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:18:00.221344059Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ABF50750B4A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:18:00.981210021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:18:00.983679735Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ABF7DE318D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:18:01.735712504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="500 kB" avg_uncomp="500 kB" max_age=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:18:01.738397544Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ABFAAE11CDA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:18:02.036568811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="254 kB" avg_comp="254 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" max_age=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:18:02.043376753Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ABFBD0EF365, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-02T03:18:04.598044993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:18:05.421804282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:18:05.426564015Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AC086B665AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:18:08.993447601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:18:08.995932924Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AC15B740FA3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T03:18:09.4191105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:18:09.42161048Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AC174D39655, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T03:18:09.598243013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:18:14.597784648Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:18:14.597790417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:18:14.597815732Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:18:15.56409324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:18:15.566839088Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AC2E31B7C1A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=info ts=2026-07-02T03:18:16.586857019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:18:16.589724405Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AC32015C92C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:18:18.389596907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:18:18.395899151Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AC38BBE4BEF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:18:18.486098567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:18:18.488565192Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AC39143824A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T03:18:19.597512444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:18:24.597884527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:18:26.172743502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:18:26.175218999Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AC55B6DFE8C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:18:26.400741477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:18:26.403267423Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AC56905D0FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:18:26.950642289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:18:26.953025981Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AC589C989F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-02T03:18:29.598085504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:18:31.198610879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:18:31.200904679Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AC686FB0724, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:18:33.330840187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="309 kB" avg_comp="309 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" max_age=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:18:33.336650267Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AC70649432E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-07-02T03:18:34.597446712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:18:34.624549436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:18:34.629360219Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AC75352E196, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:18:34.691515824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:18:34.69384953Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AC7572CFB57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:18:36.481450881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:18:36.483804651Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AC7C1DE6C3C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T03:18:38.708397527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:18:38.712876832Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AC846BA09CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:18:39.023745999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:18:39.028980086Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AC8599066DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:18:39.291675195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:18:39.294042581Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AC8695DEF3B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T03:18:39.597437814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:18:44.578315532Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T03:18:44.597306125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:18:44.597314133Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:18:44.597339331Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:18:45.134614458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:18:45.138906709Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AC9C5BFFAEB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:18:48.260792189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:18:48.266169307Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ACA8026CDFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-02T03:18:49.598126123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:18:50.391641908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:18:50.396808922Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ACAFF24F16F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:18:50.79393256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:18:50.802948245Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ACB175AB606, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:18:51.316153435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:18:51.320567557Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ACB3634A6C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:18:51.524061897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:18:51.526419376Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ACB427A97AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:18:52.009879186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:18:52.012226077Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ACB5F6E5A82, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:18:52.28716131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:18:52.289475403Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ACB6FF594E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:18:54.598238176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:18:56.052998558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:18:56.055619008Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ACC506F65A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:18:56.306254697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:18:56.308560672Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ACC5F83D5DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T03:18:58.042526681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=db0cb9c162311785 immediate=false num_chunks=1 total_comp="130 B" avg_comp="130 B" total_uncomp="48 kB" avg_uncomp="48 kB" max_age=1 labels="{stream=\"smee-client\"}"
level=error ts=2026-07-02T03:18:58.044986143Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ACCC70393A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"smee-client\"}"
level=warn ts=2026-07-02T03:18:59.597925687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:18:59.664102068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:18:59.666457343Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ACD27A872C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T03:19:01.734346831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:19:01.73697258Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ACDA3103C37, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=info ts=2026-07-02T03:19:01.956639826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:19:01.95899275Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ACDB04EA429, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:19:04.597304177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:19:08.934601371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:19:08.936841336Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ACF5037EDCD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-02T03:19:09.597339611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:19:12.365018452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:19:12.367329307Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD01CB1DF87, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:19:13.854065375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:19:13.856459524Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD07572EFA4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:19:14.597387397Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:19:14.597420879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:19:14.597436517Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:19:15.568655778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:19:15.571071869Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD0DBA5228B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:19:15.571089638Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:19:15.571107851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:19:15.572926846Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD0DBC5422B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:19:17.492484873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=db0cb9c162311785 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"smee-client\"}"
level=error ts=2026-07-02T03:19:17.494919815Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD14E50FD52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"smee-client\"}"
level=warn ts=2026-07-02T03:19:19.597784927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:19:20.593450425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:19:20.596287836Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD2072D2CD0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:19:21.353083196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:19:21.357956576Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD23492BF1F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T03:19:23.110780097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:19:23.115371446Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD29D534138, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:19:24.434015082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:19:24.438943767Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD2EC36F3E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:19:24.566268443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:19:24.568610319Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD2F3F1FE16, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T03:19:24.597606026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:19:29.597473322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:19:29.716743191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:19:29.719290791Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD426F29B35, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:19:29.74730196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:19:29.749834056Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD428C3AC08, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T03:19:33.602864126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:19:33.605731828Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD50E977909, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:19:33.71884786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:19:33.723578095Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD5159F7424, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:19:34.484582495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:19:34.486909815Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD5431FD37D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T03:19:34.597385766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:19:36.4254201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:19:36.432234419Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD5B711EE7B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:19:36.432252028Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:19:36.432275364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:19:36.441206093Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD5B79C3F41, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:19:37.647858437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:19:37.654400765Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD5FFEA65AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:19:38.87671469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:19:38.882146631Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD6490FA4C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:19:39.597668028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:19:41.944752385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:19:41.947139317Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD6FFC93A9F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:19:43.290576234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:19:43.292942553Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD750003DED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:19:44.579180352Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:19:44.597945107Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:19:44.597961237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:19:44.597974719Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:19:47.125991164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:19:47.128190951Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD8349B3BC8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:19:49.598203988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:19:49.866110927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:19:49.868503447Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD8D7EE8487, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T03:19:49.892581613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=db0cb9c162311785 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"smee-client\"}"
level=error ts=2026-07-02T03:19:49.894774895Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD8D9804EEE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"smee-client\"}"
level=info ts=2026-07-02T03:19:50.595814145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="171 B" avg_comp="171 B" total_uncomp="24 kB" avg_uncomp="24 kB" max_age=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:19:50.598314347Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD9036ECB57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:19:50.702535112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="513 B" avg_comp="513 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:19:50.704796859Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD909C8D3CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:19:50.704811156Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:19:50.896738591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:19:50.899177502Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD9155E4648, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:19:51.429601507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:19:51.432042773Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD93521A4B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:19:52.807745687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:19:52.810184699Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AD98745BE48, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T03:19:54.598000281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:19:56.320698085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:19:56.327732514Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ADA58EF6E96, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:19:58.562929381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:19:58.569584981Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ADADE8E53C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=info ts=2026-07-02T03:19:59.239453077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:19:59.241713129Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ADB069F371E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:19:59.598112009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:20:02.581467795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:20:02.583878536Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ADBCDD49835, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T03:20:04.597303147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:20:05.70845612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:20:05.71128896Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ADC883CE753, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:20:07.333333887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:20:07.337987388Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ADCE93271BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:20:09.083042654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:20:09.085375738Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ADD515A98F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T03:20:09.598069063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:20:09.751452132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:20:09.753819354Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ADD7930F4BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:20:10.332340993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:20:10.334696061Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ADD9BD1100C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:20:11.689663344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:20:11.691989661Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ADDECB7AFD3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:20:12.299461664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:20:12.301749058Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ADE110FDBE6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:20:14.597732654Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:20:14.597746576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:20:14.597766734Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:20:15.226449343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:20:15.228796635Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ADEBF86FB60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:20:17.739063859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:20:17.741342653Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ADF554A2FEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T03:20:18.062260259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:20:18.064707174Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ADF688FC7B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:20:18.441848987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:20:18.444650345Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5ADF7F345EEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-02T03:20:19.597957531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:20:22.226958017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:20:22.233629378Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE0610C4497, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:20:22.74345085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:20:22.748315844Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE07FB9C91A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:20:22.748335465Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:20:22.748357044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:20:22.755017054Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE08021A4D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:20:23.049131152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="144 kB" avg_comp="144 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" max_age=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:20:23.055203256Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE091F8C2F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:20:23.868756802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:20:23.871143765Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE0C2A730EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:20:24.597517164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:20:26.247846113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:20:26.254914419Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE150BB0E3A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:20:27.661002447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:20:27.665849095Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE1A4D45493, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T03:20:29.598145798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:20:30.621507464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:20:30.623769276Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE25523DE64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:20:30.623789164Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:20:30.623807118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:20:30.625530715Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE255410AB5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:20:30.977583484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:20:30.982241602Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE26A8011DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:20:31.216885319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:20:31.221443771Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE278C2FE08, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T03:20:34.597310587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:20:35.113613867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.4 kB" avg_comp="3.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T03:20:35.116023007Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE360E5D4DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-07-02T03:20:35.116041287Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:20:35.116061905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="116 kB" avg_comp="116 kB" total_uncomp="786 kB" avg_uncomp="786 kB" max_age=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:20:35.120260324Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE36123E822, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:20:35.890524471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:20:35.892965838Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE38F33C225, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:20:36.217902303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:20:36.220278015Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE3A2B841A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:20:36.393801834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:20:36.398745042Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE3AD5A1B13, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:20:39.183079455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:20:39.187944155Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE4539A8096, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T03:20:39.598121921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:20:39.743605077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b96e65646178f4af immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:20:39.746221217Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE474DF2C7B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-nyog\"}"
level=error ts=2026-07-02T03:20:39.74623626Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:20:39.746254389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="892 kB" avg_uncomp="892 kB" max_age=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T03:20:39.748128788Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE475012DC6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T03:20:40.681007368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:20:40.683294974Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE4ACBBEC64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:20:40.683313709Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:20:40.683334307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:20:40.685088783Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE4ACDA2138, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:20:41.725753824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:20:41.729169962Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE4EB050D2F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:20:42.069595752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=db0cb9c162311785 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"smee-client\"}"
level=error ts=2026-07-02T03:20:42.072256764Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE4FF83ABE7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"smee-client\"}"
level=info ts=2026-07-02T03:20:42.649837569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:20:42.652252674Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE522182A25, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:20:44.579224442Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:20:44.597945772Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:20:44.597955713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:20:44.597975936Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:20:49.59830672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:20:53.754753903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:20:53.758698275Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE7B812B84C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-02T03:20:54.597871037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:20:55.911452832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:20:55.913795031Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE8388AB2A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:20:56.062227569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:20:56.064609918Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE841878A77, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:20:56.454707838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:20:56.457090126Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE858EBF58C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:20:59.421734507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="80 kB" avg_comp="80 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T03:20:59.424319602Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE909C7092B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-02T03:20:59.59797658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:21:00.463869664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:21:00.466202839Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE947E25C2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:21:00.969550664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:21:00.97222077Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE9660CEEAB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:21:02.401255744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:21:02.408046934Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE9BBA174D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T03:21:02.408068878Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:21:02.408095533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="118 B" avg_comp="118 B" total_uncomp="195 kB" avg_uncomp="195 kB" max_age=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:21:02.410253414Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE9BBC563D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:21:02.853637561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:21:02.856032268Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AE9D6539EE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:21:04.598254623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:21:08.615524638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="663 B" avg_comp="663 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:21:08.617861439Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEB2DC35481, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:21:08.617879792Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:21:08.617899294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:21:08.619696495Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEB2DE1809D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:21:08.775859231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:21:08.778125317Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEB37511DA8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-07-02T03:21:08.778140604Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:21:08.918374098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:21:08.920702507Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEB3FD042EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:21:09.547251332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:21:09.549860758Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEB655101A6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:21:09.553815428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:21:09.558105242Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEB65CE220A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T03:21:09.598218133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:21:09.809785895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:21:09.813041147Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEB74F3FBB6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:21:10.351209656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:21:10.353574029Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEB9536ED50, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:21:10.353593493Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:21:10.35361216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:21:10.355333239Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEB9556469D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:21:10.823682218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:21:10.825712805Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEBB15EC17E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T03:21:10.825727511Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:21:10.825747659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="67 kB" avg_comp="67 kB" total_uncomp="559 kB" avg_uncomp="559 kB" max_age=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:21:10.827482137Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEBB17B0EE7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:21:11.304065137Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE5AEBCDE18443, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T03:21:11.428435941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:21:11.43075515Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEBD56CB87C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:21:11.430773035Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:21:11.430793121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:21:11.432614938Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEBD58B36DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:21:14.597511215Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:21:14.597528157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:21:14.597545837Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:21:14.628762251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:21:14.631102393Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEC942DBBA8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:21:15.012318818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:21:15.016772161Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AECAB2AC156, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:21:15.282995848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:21:15.292703571Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AECBB9C9B31, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:21:16.554860098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:21:16.559931077Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AED07242010, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:21:18.50494394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:21:18.507313767Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AED7B38721C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:21:19.142281837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:21:19.144661872Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEDA1352EF8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:21:19.14468023Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:21:19.144698516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:21:19.146523163Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEDA1548A9D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:21:19.42042157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:21:19.422858367Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEDB1CB08F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:21:19.422873622Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:21:19.445057235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:21:19.447233251Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEDB33F94C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:21:19.597517612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:21:20.744450916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:21:20.746804855Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEE00B406AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:21:22.185388902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:21:22.187938867Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEE56987ED8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:21:22.313452418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:21:22.315804807Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEE5E3A17CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:21:23.277072473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:21:23.279742375Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEE97AAAF2B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:21:23.456185804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=db0cb9c162311785 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"smee-client\"}"
level=error ts=2026-07-02T03:21:23.458525243Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEEA255E5AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"smee-client\"}"
level=info ts=2026-07-02T03:21:24.359110779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:21:24.363291312Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEED844008F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T03:21:24.598234108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:21:26.003588461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:21:26.008759514Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEF3A55D665, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:21:28.513923214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:21:28.51632501Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEFCFCDBC8C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:21:28.644837869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:21:28.647197576Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AEFD79A49A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T03:21:29.597909646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:21:31.189761107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:21:31.192141916Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF06F4B9610, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-07-02T03:21:31.192160713Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:21:31.192182915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="315 kB" avg_comp="315 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" max_age=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:21:31.198863874Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF06FB3B8C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:21:31.207866731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:21:31.212189018Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF0708057EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:21:31.409782119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:21:31.412211944Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF07C68D286, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:21:34.597843698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:21:35.38482425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:21:35.38728458Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF1695756CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:21:35.477506329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="80 kB" avg_comp="80 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T03:21:35.480044719Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF16EDD9E3D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T03:21:35.990545622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:21:35.99299676Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF18D71A615, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:21:36.351890161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:21:36.35410758Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF1A2F9EC34, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:21:38.007440785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:21:38.009776778Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF205A73D28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:21:38.009795178Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:21:38.00981469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:21:38.011599Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF205C6351D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:21:39.597522973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:21:41.390041397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:21:41.392424543Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF2CF47941B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:21:44.57913392Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:21:44.597850749Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:21:44.59788048Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:21:44.597942163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:21:45.491383072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:21:45.493693329Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF3C3BC3E62, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:21:48.55104489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:21:48.554221326Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF47A2619FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-07-02T03:21:49.597984219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:21:50.514671879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:21:50.520032595Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF4EF52EE8F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:21:50.520048431Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:21:50.520068871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:21:50.521899653Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF4EF72310F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:21:51.979242365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:21:51.981619165Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF546717924, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:21:52.955085643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:21:52.957378408Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF5809B2AE2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:21:53.697732455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:21:53.700739228Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF5ACE80397, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:21:54.321858146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:21:54.326713623Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF5D237EA62, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:21:54.597809301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:21:56.236837171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:21:56.241280641Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF644575FD7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:21:57.193055694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:21:57.198004616Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF67D5E2D64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:21:58.07120494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:21:58.073539338Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF6B18DE2B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:21:59.591733079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:21:59.594143854Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF70C2EE288, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T03:21:59.59817284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:22:01.01945666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:22:01.022512243Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF7614E09FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:22:01.381588154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:22:01.39087942Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF77746AB33, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:22:01.676723692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:22:01.681706437Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF7889ACC1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:22:03.94837817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:22:03.950769026Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF80FDC5B84, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T03:22:04.597530302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:22:06.554756109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=db0cb9c162311785 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"smee-client\"}"
level=error ts=2026-07-02T03:22:06.557120786Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF8AB369FD6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"smee-client\"}"
level=info ts=2026-07-02T03:22:06.597593625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:22:06.60083144Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF8ADC61767, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:22:07.883935617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:22:07.886252568Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF8FA6F0697, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:22:09.192249614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:22:09.19448852Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF9486978B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:22:09.562300992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:22:09.56470925Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF95E7A40B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:22:09.598170642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:22:09.623333442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:22:09.62555693Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AF9621B9B10, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:22:13.730929139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:22:13.733630248Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AFA56F45203, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:22:14.597238119Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:22:14.597276738Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:22:14.597280902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:22:14.603436366Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-07-02T03:22:14.605499742Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007377055s
level=info ts=2026-07-02T03:22:15.952925947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="80 kB" avg_comp="80 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T03:22:15.955681834Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AFADB66BECB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T03:22:17.095307834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:22:17.099496671Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AFB1F95EAD9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:22:17.099517503Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:22:17.099539934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:22:17.108625874Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AFB2021938C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:22:17.438463185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:22:17.443696088Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AFB3417035A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T03:22:19.598065615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:22:22.221324306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:22:22.224569785Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AFC510E18F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-07-02T03:22:24.597316932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:22:27.753212375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:22:27.761138671Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AFD9B109788, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:22:28.929518021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:22:28.935024417Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AFDE10965C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T03:22:29.597886771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:22:29.647049418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:22:29.649333319Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AFE0B9DF32D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:22:29.754564918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:22:29.756821077Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AFE1205B49C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:22:29.818680181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="920 B" avg_comp="920 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:22:29.820936464Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AFE15D89F1D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:22:29.820955157Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:22:29.820976081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:22:29.82278611Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5AFE15F6B462, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T03:22:34.597457873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:22:38.253797783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:22:38.256559848Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B000CA223D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:22:39.598194884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:22:39.859087747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:22:39.86139644Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B006C4D4B3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:22:40.077760977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:22:40.080029594Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B007955A510, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:22:44.579059337Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:22:44.597000475Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:22:44.597001584Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T03:22:44.597030053Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:22:44.597151575Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-07-02T03:22:44.597161804Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T03:22:44.597179844Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.121µs success=true
level=warn ts=2026-07-02T03:22:44.597279947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:22:46.001981803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:22:46.004298367Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B01DA721B38, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:22:46.776642335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:22:46.778939324Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B02089F36E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:22:48.047863063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=db0cb9c162311785 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"smee-client\"}"
level=error ts=2026-07-02T03:22:48.050270082Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B025465FEF0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"smee-client\"}"
level=warn ts=2026-07-02T03:22:49.598146148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:22:49.68645182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:22:49.688780368Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B02B60EF1F0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:22:49.85083326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:22:49.854196883Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B02BFDE4BCA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:22:49.999048705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:22:50.001297287Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B02C8B0BEF7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:22:50.012323319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:22:50.016325304Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B02C99752C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:22:50.198987061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:22:50.204463425Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B02D4CA5030, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:22:50.504154841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:22:50.513457921Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B02E736B860, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:22:51.930586239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:22:51.935933801Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B033BFF8636, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-07-02T03:22:54.597438522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:22:55.570179639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:22:55.575097639Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B0414E6D8FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:22:56.279009319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:22:56.281521569Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B043F03E1B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:22:57.134931076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:22:57.137338661Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B047206DFF7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:22:57.981127116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:22:57.983876804Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B04A47943F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:22:58.881568811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:22:58.883936061Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B04DA222D03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T03:22:59.597659864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:23:00.725750048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:23:00.728148667Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B05480ED332, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:23:01.029084909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="80 kB" avg_comp="80 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T03:23:01.03166295Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B055A238524, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T03:23:02.290745693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:23:02.294941876Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B05A570CD67, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:23:02.466373012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:23:02.471047412Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B05AFF0D853, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:23:02.832039101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:23:02.834741278Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B05C59EF92F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:23:04.512610078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:23:04.517279413Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B0629E7F73C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T03:23:04.597902895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:23:05.118840496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:23:05.121195766Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B064DE6E306, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T03:23:09.598056114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:23:14.597658465Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:23:14.597671902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:23:14.597688767Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:23:16.134565224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:23:16.139070992Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B08DE9E18F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:23:17.147439573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:23:17.149944444Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B091ADE6B14, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:23:18.428444273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:23:18.430754678Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B096736ECEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:23:19.597921068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:23:24.59832268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:23:27.034596032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:23:27.036854876Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B0B682D914F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:23:29.597687901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:23:29.603835572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=db0cb9c162311785 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"smee-client\"}"
level=error ts=2026-07-02T03:23:29.606040578Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B0C0150CDEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"smee-client\"}"
level=info ts=2026-07-02T03:23:32.08950797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:23:32.091841941Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B0C957A0400, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:23:32.778863587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:23:32.781166808Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B0CBE918BB5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T03:23:34.598093386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:23:35.088450859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:23:35.091394558Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B0D484308E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:23:36.301871052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:23:36.30769207Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B0D90C1F2AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:23:36.723893401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:23:36.728271414Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B0DA9D56ED4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:23:37.129338604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:23:37.134158244Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B0DC2062104, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:23:39.597819856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:23:43.307114169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:23:43.309799976Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B0F321C7C64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:23:43.479345876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:23:43.48239509Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B0F3C6778A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:23:44.29547543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:23:44.300260795Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B0F6D2808E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:23:44.353315985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:23:44.355685769Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B0F7075BCD2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:23:44.57832648Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:23:44.597740422Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:23:44.597739074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:23:44.597767098Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:23:45.251419005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:23:45.254134815Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B0FA6044431, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:23:45.686445418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:23:45.692046404Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B0FC010AD03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:23:46.656433306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:23:46.658875512Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B0FF9BE73C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:23:48.266777795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:23:48.27351305Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B1059FBEF99, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:23:48.858101479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:23:48.862550355Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B107D176C14, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:23:49.299963461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:23:49.304875331Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B1097738250, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:23:49.372040458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:23:49.376738859Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B109BBC4A1F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T03:23:49.597449539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:23:50.636513922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="80 kB" avg_comp="80 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T03:23:50.638966503Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B10E6F87A00, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T03:23:54.563644881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:23:54.566039337Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B11D10B6173, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-02T03:23:54.597623234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:23:54.642678577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:23:54.644870817Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B11D5BE8AE6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:23:55.803327299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:23:55.805781568Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B121AF04299, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T03:23:59.59753003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:24:01.619219543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:24:01.621724743Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B1375986450, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:24:02.998809891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:24:03.001120393Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B13C7D11385, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:24:04.59790324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:24:05.869100742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:24:05.871596473Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B1472E82230, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T03:24:09.597762349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:24:14.045041738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:24:14.047363474Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B165A39E0F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T03:24:14.597770999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:24:14.597812863Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:24:14.597837961Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:24:16.533764411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:24:16.536183026Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B16EE918640, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:24:16.536201934Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:24:16.53621964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:24:16.538243248Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B16EEB33DFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:24:17.066834339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:24:17.069074968Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B170E557F87, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:24:19.076394146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:24:19.07920416Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B178624C7CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-07-02T03:24:19.598128037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:24:20.135149163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:24:20.139843971Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B17C55CFF2F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:24:23.075447655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:24:23.077868767Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B18747AED3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:24:23.24723406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:24:23.251758229Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B187ED8AC75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T03:24:24.597683502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:24:26.305717909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:24:26.310684463Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B19352B742A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:24:26.411029383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:24:26.413377157Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B193B4B5A2D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:24:27.966729299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=db0cb9c162311785 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"smee-client\"}"
level=error ts=2026-07-02T03:24:27.968979003Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B1998040FB0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"smee-client\"}"
level=info ts=2026-07-02T03:24:28.025218974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:24:28.027694175Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B199B82BB75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:24:29.21403666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:24:29.216264824Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B19E25D49F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:24:29.228308977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:24:29.230743363Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B19E33A4BCD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-07-02T03:24:29.230759144Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:24:29.230782088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:24:29.235181368Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B19E37F3705, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-02T03:24:29.597953146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:24:30.308296365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:24:30.313037617Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B1A23BAC373, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:24:30.790157578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:24:30.792924921Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B1A40561368, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:24:33.111451315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="513 B" avg_comp="513 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:24:33.116015124Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B1ACACEB0D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:24:33.416874319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:24:33.419215573Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B1ADCE0D63F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T03:24:34.597919966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:24:35.757974384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:24:35.760318539Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B1B6869CC81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:24:36.285066251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:24:36.28835634Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B1B87D7AEDF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:24:36.419742813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:24:36.427365935Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B1B902CF72B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T03:24:39.597452806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:24:39.848955728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:24:39.854233793Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B1C5C6CE934, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:24:40.963004514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:24:40.965781319Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B1C9EAF00E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:24:42.090575188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:24:42.095442382Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B1CE2046091, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T03:24:44.00859735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="80 kB" avg_comp="80 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T03:24:44.011180102Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B1D54325C98, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T03:24:44.561191196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:24:44.563717182Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B1D7521950C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:24:44.578318711Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:24:44.597807397Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:24:44.59782951Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:24:44.597859901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:24:48.572162688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:24:48.576414566Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B1E6450B385, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T03:24:49.597431196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:24:53.814038935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:24:53.820101705Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B1F9CDD277D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:24:54.214201541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:24:54.216555352Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B1FB47E93CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T03:24:54.597413726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:24:57.457708367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:24:57.46004645Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B2075D282D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:24:59.597569724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:25:03.715462024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:25:03.717804092Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B21EAD07C22, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:25:04.597795316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:25:05.015113878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:25:05.017557583Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B223848D4BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:25:05.53874213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:25:05.541460421Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B225782D0F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:25:05.54147876Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:25:05.54150019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:25:05.547785679Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B2257E47A0D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:25:06.447902924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:25:06.453494976Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B228DDE7BD2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T03:25:09.598138517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:25:09.656268508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:25:09.660830807Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B234D0A2169, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:25:10.320845061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:25:10.325296378Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B2374A6D317, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:25:14.597583796Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:25:14.597618161Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:25:14.597639253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:25:15.08905997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:25:15.091536376Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B2490BC3512, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:25:16.064975378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:25:16.067270819Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B24CAE5FBE7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:25:17.728738955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=db0cb9c162311785 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"smee-client\"}"
level=error ts=2026-07-02T03:25:17.731061991Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B252E11E655, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"smee-client\"}"
level=error ts=2026-07-02T03:25:17.731077613Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:25:17.731096445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="201 B" avg_comp="201 B" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" max_age=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:25:17.732968748Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B252E312228, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:25:18.225359919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:25:18.228165326Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B254BB32B59, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:25:18.341346497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:25:18.346614042Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B2552C20D69, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:25:18.929079628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:25:18.933869114Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B2575C16001, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:25:19.59786726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T03:25:20.170809356Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE5B25BF7F5D8E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T03:25:20.973550953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:25:20.975893443Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B25EF7969FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:25:22.10271064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:25:22.106079699Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B2632CA8CF2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:25:24.432884858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:25:24.435241768Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B26BDAAE0E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:25:24.597518758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=warn ts=2026-07-02T03:25:24.597573547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T03:25:24.600289551Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B26C77E01BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:25:24.676442753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:25:24.679231567Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B26CC35D12F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:25:24.866384425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:25:24.870783158Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B26D7A0D953, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:25:24.93985399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:25:24.942225368Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B26DBE2B5F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:25:29.360795446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:25:29.363198428Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B27E36549AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T03:25:29.598050533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:25:31.177826559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:25:31.180282718Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B284FB38FAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:25:32.203316757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:25:32.207491412Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B288CEE2E1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T03:25:34.598104262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:25:36.281080465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:25:36.287278299Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B29801AAD45, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:25:36.326325687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:25:36.33294458Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B2982D4566C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:25:38.749424972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:25:38.754197222Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B2A132456DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:25:39.404086053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:25:39.406301683Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B2A3A0459FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T03:25:39.597792922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:25:42.277581257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:25:42.279907491Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B2AE54ABC20, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:25:42.896670389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="80 kB" avg_comp="80 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T03:25:42.899176572Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B2B0A335646, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T03:25:44.44951493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:25:44.451834614Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B2B66C0800E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:25:44.578316127Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:25:44.597848773Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:25:44.597876871Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:25:44.597903855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:25:47.159990608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:25:47.162344639Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B2C084FCA7F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:25:49.48524443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:25:49.487859688Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B2C92EC2987, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-07-02T03:25:49.59791991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:25:53.576445829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:25:53.581759597Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B2D86F0159C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T03:25:54.597433756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:25:55.928877708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:25:55.933348136Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B2E1319F262, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:25:57.041212824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:25:57.045791817Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B2E556881C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:25:58.281283195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:25:58.283643051Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B2E9F2FE730, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:25:59.597587806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:26:01.009956872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:26:01.012267936Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B2F41D43BBE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T03:26:01.567105139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:26:01.569423744Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B2F63095490, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:26:04.59817136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:26:05.830055731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:26:05.832842436Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3061289635, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:26:06.265978543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:26:06.27083028Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B307B42D819, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:26:09.139963021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:26:09.142458255Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B31266C4CB6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:26:09.479507525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:26:09.482883284Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B313AABF191, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T03:26:09.598259509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:26:09.922536661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:26:09.924866496Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B31550EA708, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:26:11.324057116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:26:11.326467838Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B31A8991E28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-07-02T03:26:11.326489753Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:26:11.32651306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:26:11.328423188Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B31A8BA563F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:26:12.18599709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:26:12.18875388Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B31DBFC1780, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:26:13.788377706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:26:13.790835035Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B323B7CC33A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:26:13.79085663Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:26:13.790879469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:26:13.792741539Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B323B9C6BE6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:26:14.597484005Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:26:14.597520378Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:26:14.597538333Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:26:15.297560539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:26:15.299878742Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B32956FF0C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:26:15.29989652Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:26:15.299917549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:26:15.301670851Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B32958DA7CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:26:15.83453298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="663 B" avg_comp="663 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:26:15.836947268Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B32B571C2BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:26:17.03523935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:26:17.038065951Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B32FD0AB465, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-07-02T03:26:19.598164956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:26:22.525870992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:26:22.535315887Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3444B44366, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:26:22.535336711Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:26:22.535363166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:26:22.544569462Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3445419E2A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:26:23.067064291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:26:23.073721623Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3464CA9DBC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T03:26:24.598005586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:26:27.284237583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:26:27.289127486Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B35600CB9A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:26:28.110447031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:26:28.113077773Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B359124FF8E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:26:29.597441793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:26:30.269277888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:26:30.271500015Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3611D0D0C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:26:31.302235599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:26:31.304705322Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B364F6477A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:26:33.432284989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:26:33.434561303Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B36CE59B912, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:26:33.776659229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:26:33.779023593Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B36E2E0E5EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:26:34.347207894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:26:34.349468327Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3704E1DCA3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T03:26:34.597605915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:26:35.764524797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:26:35.772423737Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3759B0DE40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:26:35.961824622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="80 kB" avg_comp="80 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T03:26:35.966939295Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3765483641, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T03:26:37.025444138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:26:37.027742755Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B37A483A51D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-02T03:26:37.771674444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:26:37.77424179Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B37D1022186, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:26:39.598268032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:26:39.93344949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:26:39.935861358Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3851D9B088, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:26:44.578542354Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:26:44.597088269Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:26:44.597115792Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:26:44.598218473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:26:46.98500755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:26:46.987881771Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B39F62F6EC4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:26:48.913744411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:26:48.91945004Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3A69505AF9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:26:49.329456697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:26:49.332006049Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3A81E83AEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:26:49.58263063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:26:49.584898347Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3A90FADE95, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T03:26:49.597960272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:26:50.821562075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:26:50.823981316Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3ADAD54E27, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:26:52.173888299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:26:52.176282306Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3B2B711A46, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:26:53.432770578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:26:53.435035431Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3B7678F6B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:26:53.939452794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:26:53.942124353Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3B94AEF877, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T03:26:54.597360903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:26:54.940702894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:26:54.945117509Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3BD07AF7C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:26:54.945136784Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:26:54.94515886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:26:54.951837589Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3BD0E1CF2D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:26:56.280367575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:26:56.285346762Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3C205C63A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:26:59.21939655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:26:59.2222309Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3CCF6A7CC1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:26:59.447777867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:26:59.453520991Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3CDD26175E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:26:59.453541736Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:26:59.453562225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:26:59.455840423Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3CDD5431CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T03:26:59.597423609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:26:59.744589094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:26:59.74685887Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3CEEAF2777, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:27:00.447452171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:27:00.450224423Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3D189945C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:27:00.450241665Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:27:00.45026059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:27:00.452518398Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3D18BC2360, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:27:00.839604514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:27:00.841956539Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3D2FF3EF10, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:27:01.542449106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:27:01.545364326Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3D59E24D4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:27:01.583838411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:27:01.592915557Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3D5CB6C0DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:27:02.491266879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:27:02.495980633Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3D928A1456, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:27:04.598125674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:27:07.252229366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:27:07.256645412Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3EAE4CB1BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T03:27:09.597721008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:27:10.994306851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:27:10.999717527Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3F8D6399CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:27:12.072270026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:27:12.074764829Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3FCD7B0C8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:27:12.074781157Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:27:12.074798953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:27:12.076609484Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B3FCD9A60E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:27:13.625886775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:27:13.630113317Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B402A309428, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:27:13.994090512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:27:13.999688852Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B40402C5822, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:27:14.597250406Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:27:14.597283247Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:27:14.597272308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:27:14.598373694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:27:14.600670141Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B406409C42D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:27:14.600683746Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:27:14.600703076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:27:14.602490064Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4064283BE9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:27:14.603254085Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-07-02T03:27:14.605149759Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007796065s
level=warn ts=2026-07-02T03:27:19.5981896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:27:21.755570422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="920 B" avg_comp="920 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:27:21.7578364Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B420EA4585A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:27:22.446121276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="80 kB" avg_comp="80 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T03:27:22.448674912Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4237CE27A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T03:27:22.448691794Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:27:22.448711727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="124 B" avg_comp="124 B" total_uncomp="58 kB" avg_uncomp="58 kB" max_age=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:27:22.450588244Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4237F0CF25, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:27:23.773915363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:27:23.776268857Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4286F2E289, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-02T03:27:23.776285419Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:27:23.776305815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="158 B" avg_comp="158 B" total_uncomp="207 kB" avg_uncomp="207 kB" max_age=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:27:23.778318914Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B42871419F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-02T03:27:24.59738581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:27:27.177595795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:27:27.179880338Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4351D1BE19, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-07-02T03:27:27.17989843Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:27:27.179918463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="465 B" avg_comp="465 B" total_uncomp="243 kB" avg_uncomp="243 kB" max_age=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:27:27.182039522Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4351F4EC47, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:27:27.49190522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:27:27.494270903Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B43648E5392, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:27:27.550338708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:27:27.55261944Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4368096DB3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:27:27.842674444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:27:27.845023411Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B437977B411, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:27:27.845040123Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:27:27.845058315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:27:27.846837677Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4379953137, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:27:29.598290206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:27:32.407594438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:27:32.41021004Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B448990EF74, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:27:32.718982877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:27:32.721425564Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B449C1DE662, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:27:33.759299315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:27:33.764205467Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B44DA46A715, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T03:27:34.597693619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:27:35.36392137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:27:35.370266895Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B453A00790A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:27:37.047918662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:27:37.050608052Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B459E2623E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:27:37.050625129Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:27:37.050645245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:27:37.052508959Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B459E4817BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:27:37.165725872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:27:37.167887549Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B45A5274C62, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:27:39.169280031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:27:39.171656287Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B461C95D31C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:27:39.470446264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:27:39.4726873Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B462E8798B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:27:39.472705034Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:27:39.472724273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:27:39.474509154Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B462EA582FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T03:27:39.598070176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:27:40.316594739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:27:40.318860539Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4660F6DF98, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:27:42.351920453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:27:42.358901059Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B46DA8E4617, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:27:42.668979346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="87 kB" avg_uncomp="87 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T03:27:42.673951416Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B46ED55C97D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-07-02T03:27:42.673969599Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:27:42.673992074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="124 B" avg_comp="124 B" total_uncomp="47 kB" avg_uncomp="47 kB" max_age=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:27:42.675782169Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B46ED7588FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:27:43.195342348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:27:43.197538537Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B470C8C7A4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:27:43.34178921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:27:43.345111283Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B47154A86EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:27:43.931312096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:27:43.933850998Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B47386E2090, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:27:44.540757238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:27:44.543540864Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B475CC5B216, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:27:44.579257607Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:27:44.597267307Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:27:44.597298016Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:27:44.597310132Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-07-02T03:27:44.597322709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:27:44.597474871Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-07-02T03:27:44.597483157Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T03:27:44.597507868Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.458µs success=true
level=info ts=2026-07-02T03:27:45.029082277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:27:45.034254658Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B477A0365F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:27:45.32300654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:27:45.325542223Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B478B6030EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:27:45.388864467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:27:45.391605299Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B478F518E51, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:27:46.889585711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:27:46.894273737Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B47E8E2836B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T03:27:49.597454697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:27:49.917542759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:27:49.919943931Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B489D3AC268, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:27:49.919961557Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:27:49.919980792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:27:49.921813217Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B489D5A6FAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:27:50.232311514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:27:50.234733351Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B48AFFE53B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:27:53.587447211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:27:53.590354121Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4978003E9D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:27:53.590370864Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:27:53.590394634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:27:53.595057931Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B49784B0706, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-07-02T03:27:54.598123377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:27:55.2302604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:27:55.235347355Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B49DA0A97A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:27:55.235374015Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:27:55.235399232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:27:55.237291956Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B49DA2E1BAA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:27:56.622855731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:27:56.625443661Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4A2CE7532C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:27:57.076534619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:27:57.078885691Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4A47EF8471, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:27:59.04807861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:27:59.050281733Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4ABD72011C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-07-02T03:27:59.59776177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:28:00.616958146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:28:00.619331193Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4B1AF7C6B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:28:03.713440048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:28:03.715777395Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4BD386A01A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T03:28:04.597694048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:28:04.872668654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:28:04.877358764Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4C18C20FF7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:28:07.443597934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:28:07.449995317Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4CB219550D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:28:08.218813936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:28:08.221114667Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4CE011AC0F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:28:08.310314236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:28:08.313118528Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4CE58D9EA8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:28:08.370242143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:28:08.375419118Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4CE9430E3B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-02T03:28:09.597616667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:28:12.361625252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:28:12.364038793Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4DD6FEDA6A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:28:12.443975882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:28:12.446294927Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4DDBE87FEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:28:14.597516586Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:28:14.597552854Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:28:14.597572028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:28:15.756699267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:28:15.758883988Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4EA15AC9A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:28:16.443681426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:28:16.44618019Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4ECA51DECE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T03:28:19.598111338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:28:20.758205597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:28:20.760823358Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4FCB7D1A43, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:28:21.325399086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:28:21.327897051Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4FED49EB70, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:28:21.551259784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:28:21.553781411Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4FFAC126CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:28:21.55379852Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:28:21.553819953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:28:21.555820253Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B4FFAE27ACE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:28:23.354769883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:28:23.357300363Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B50663F96FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T03:28:24.597836086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:28:24.896451248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:28:24.901035606Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B50C243527A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:28:25.99898366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:28:26.004882785Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B51040FE0C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:28:26.09007841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:28:26.092820208Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B51094C4D23, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:28:27.000435767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:28:27.004958253Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B513FABE858, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:28:27.683719761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:28:27.690336125Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B516886479E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T03:28:29.597295542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:28:32.306528866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:28:32.311307501Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B527BF44501, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:28:32.701453575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:28:32.704160672Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B52935F230F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-07-02T03:28:34.597330213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:28:34.89645777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:28:34.901075972Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5316506A0A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:28:35.028448927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:28:35.030756962Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B531E0B434E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:28:35.338039802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:28:35.340536443Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B533081429E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:28:35.340552282Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:28:35.340570478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:28:35.34229708Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5330A0B1EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:28:37.089645305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:28:37.091886625Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5398E65583, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:28:39.598148703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:28:39.638296058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:28:39.641050673Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5430D7F846, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:28:41.707182666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:28:41.71281731Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B54AC487008, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:28:43.298990319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:28:43.301551524Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B550B0392A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:28:43.635602412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:28:43.638361914Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B551F15F5AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:28:44.578375135Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:28:44.597072048Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:28:44.597101791Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:28:44.598191284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:28:48.515068018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:28:48.517363528Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5641E9F99C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:28:48.517379714Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:28:48.51739856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:28:48.519149478Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5642071D75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:28:48.641731958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:28:48.643929323Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5649753882, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:28:49.597299085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:28:54.347640345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:28:54.350023056Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B579D8F6D93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:28:54.353064725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:28:54.354975279Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B579DDDC42F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:28:54.598169523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:28:57.283053349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:28:57.285467072Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B584C879A05, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T03:28:59.597296834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:28:59.616443287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:28:59.618782164Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B58D79B4E46, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:28:59.696984404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:28:59.699268035Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B58DC681739, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:29:01.74365344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:29:01.746002473Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B595665E6CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:29:03.086642298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:29:03.090867992Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B59A68E506F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:29:03.090887034Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:29:03.090908947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:29:03.098467899Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B59A7041FDA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:29:04.004823903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:29:04.00951435Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B59DD508741, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:29:04.59793403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:29:08.284923574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:29:08.289555063Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5ADC6C2DCB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:29:09.500292664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:29:09.505033774Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5B24DF4891, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T03:29:09.59796112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:29:12.533046131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:29:12.535309394Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5BD97E8571, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:29:12.617787038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:29:12.620004345Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5BDE8A4F7E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:29:13.871821727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:29:13.874390759Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5C294DDD6E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:29:14.161972801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:29:14.164124021Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5C3A94E584, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T03:29:14.597901602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:29:14.597900388Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:29:14.597931681Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:29:15.024452593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:29:15.026638044Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5C6DFD371A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:29:15.052690243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:29:15.054871143Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5C6FAB7D8E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:29:16.526459099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:29:18.063505733Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5D22FE6562, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:29:18.825821186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:29:18.832563981Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5D50D6A4B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T03:29:19.597450297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:29:22.091529044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:29:22.096806548Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5E1366013F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:29:22.423639762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:29:22.428875624Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5E27309893, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:29:22.505069947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:29:22.509660827Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5E2C00E9C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:29:23.480449758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:29:23.482722861Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5E66018A2F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:29:24.168678219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:29:24.171463192Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5E8F0FAE6E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-07-02T03:29:24.597528346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:29:29.422826149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:29:29.428311237Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B5FC8627436, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=warn ts=2026-07-02T03:29:29.597249874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:29:30.779649145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:29:30.782067064Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B601915A25B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:29:32.908096565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:29:32.910484342Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B6097F1F1DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:29:33.477942617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:29:33.47990071Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B60B9E55B4B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T03:29:34.598168827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:29:36.132922836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:29:36.136369673Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B61582C8989, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:29:39.478367567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:29:39.481201954Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B621F93DE4C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-02T03:29:39.598257363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T03:29:43.248647965Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE5B630027CBC2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T03:29:44.578975744Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:29:44.596949966Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:29:44.596986777Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:29:44.59806269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:29:45.404300839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:29:45.406710266Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B6380C73B93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:29:48.384841203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:29:48.387194036Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B64326E37CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:29:49.026030071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:29:49.030218704Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B6458C2AB5F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T03:29:49.597304336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:29:49.996754305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:29:50.001343145Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B6492A3BF87, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:29:50.493501054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:29:50.49578405Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B64B01CE4E7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:29:52.190018892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:29:52.19220383Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B65153BD758, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:29:53.704324484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:29:53.706781815Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B656F7F74CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-07-02T03:29:54.598034617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:29:58.611204976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:29:58.616057613Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B66941DD80E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T03:29:59.597510865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:30:00.263537393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:30:00.269795602Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B66F6AEDB84, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:30:02.82689856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:30:02.829466209Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B678F40679C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:30:02.966113399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:30:02.96837609Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B67978A8BA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T03:30:04.597275716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:30:05.854719073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:30:05.857681567Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B6843C069AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:30:09.348285391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:30:09.353274302Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B691419C041, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T03:30:09.598167329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:30:10.734712516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:30:10.742534515Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B6966EA25DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:30:11.039450614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:30:11.044428055Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B6978E83138, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:30:11.447521132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:30:11.449918576Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B699113E118, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:30:11.919247403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:30:11.921563037Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B69AD300A0C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:30:12.13806427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="513 B" avg_comp="513 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:30:12.140584775Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B69BA3CA395, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:30:13.356279301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:30:13.358587499Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B6A02D6F7EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:30:14.542746402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:30:14.545701983Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B6A499985D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:30:14.597879681Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:30:14.597914474Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:30:14.597977329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:30:14.800743212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:30:14.805689057Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B6A5919035A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:30:19.021444474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:30:19.026025119Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B6B54A65045, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:30:19.598149772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:30:21.838391127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:30:21.840732344Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B6BFC69BF65, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:30:24.597368577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:30:26.769034779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:30:26.771460325Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B6D224F0498, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:30:29.547581458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:30:29.550070574Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B6DC7EC08D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T03:30:29.598000905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:30:34.198001303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:30:34.201612898Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B6EDD208019, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T03:30:34.597767305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:30:34.724148154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:30:34.726546337Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B6EFC784CF0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:30:36.00928329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:30:36.011674223Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B6F491161AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:30:37.270679789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:30:37.273311206Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B6F944240DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-02T03:30:39.597735714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:30:42.466519261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:30:42.468925844Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B70C9F30B05, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:30:43.321398029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:30:43.326240982Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B70FD0DE10D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:30:43.54267273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:30:43.551394116Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B710A77A134, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:30:44.00684839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:30:44.011540361Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7125E66D83, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:30:44.055560307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:30:44.057676997Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7128A6D90E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:30:44.237154161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:30:44.239464213Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B71337BB357, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:30:44.579196249Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:30:44.597011298Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:30:44.597036075Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:30:44.598146218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:30:49.597610783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:30:50.876653892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:30:50.87957151Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B72BF4324C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:30:53.892182463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:30:53.897323451Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B737322BCA4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T03:30:54.597305078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:30:54.711604051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:30:54.713921971Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B73A3CF8509, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:30:54.715881968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:30:54.717701033Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B73A40B58D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:30:54.839829456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:30:54.842488762Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B73AB7923C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-07-02T03:30:54.842506216Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:30:54.842524343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:30:54.847236346Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B73ABC3B91C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:30:55.738734216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:30:55.74376781Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B73E1315ED5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:30:55.902281704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:30:55.904542734Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B73EAC7663C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:30:56.996802215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:30:56.999222891Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B742C069C76, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:30:56.999241113Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:30:56.999261839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:30:57.001179036Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B742C26C578, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T03:30:59.598289495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:31:02.016081938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="663 B" avg_comp="663 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:31:02.018637937Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7557335FCA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:31:02.582262962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:31:02.585164209Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7578F9BF3A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:31:03.532039083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:31:03.536542482Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B75B1AE6348, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:31:04.070267617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:31:04.072623271Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B75D1A21D7D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:31:04.598252107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:31:05.22104324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:31:05.225809477Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B76165C9881, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-02T03:31:09.597796225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:31:10.405959379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:31:10.410741418Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B774B6A2358, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:31:10.41575989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:31:10.422279592Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B774C1B0346, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:31:13.478051437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:31:13.482648763Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B780283A9C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:31:14.134765688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:31:14.137518982Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B78298D2E92, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:31:14.597683259Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:31:14.597717693Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:31:14.597711869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:31:15.38000743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:31:15.383437873Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7873C23075, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:31:16.411264575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:31:16.413697419Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B78B13808B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:31:18.377361516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:31:18.3797073Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B792666B746, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T03:31:19.597420395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:31:23.791366737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:31:23.793778105Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7A691A62A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:31:24.188872651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:31:24.191170923Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7A80CB05FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:31:24.598134935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:31:24.740453089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:31:24.744856409Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7AA1CB8B51, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:31:26.719358902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:31:26.726953723Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7B17ECD155, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:31:27.921184466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:31:27.923731857Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7B5F444478, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:31:28.338775549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:31:28.341293031Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7B78285A0C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-07-02T03:31:29.597317958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:31:32.440934122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:31:32.443718348Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7C6CAACFEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:31:32.532919293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:31:32.537083272Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7C723FB199, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:31:34.2018188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:31:34.206549825Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7CD5C0D399, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T03:31:34.598009777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:31:36.265328269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:31:36.268376865Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7D50A4B66D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:31:37.712350957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:31:37.717224512Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7DA7024CB6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:31:39.421887647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:31:39.424315857Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7E0CC1E753, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T03:31:39.59744383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:31:41.058314162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:31:41.061281619Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7E6E51CE0C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:31:41.061299182Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:31:41.061321033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:31:41.065989999Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7E6E9E446D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:31:43.199578856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:31:43.204333459Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7EEE10BBAB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:31:44.578361136Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:31:44.597052735Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:31:44.597081225Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:31:44.598184389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:31:44.681670397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:31:44.684070791Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7F46443459, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:31:46.929690095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:31:46.932104181Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7FCC40C12F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:31:47.621138951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:31:47.624180944Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B7FF581429A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:31:48.821821105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:31:48.826657059Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B803D2DDC46, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:31:49.438306996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:31:49.440579739Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B8061C63C48, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T03:31:49.597769608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:31:52.08992909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:31:52.092850091Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B80FFDA920F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-07-02T03:31:54.597805879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:31:55.68952264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:31:55.694648102Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B81D68B4F74, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:31:58.734327052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="920 B" avg_comp="920 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:31:58.736624406Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B828BDD509D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:31:59.09364786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:31:59.095857486Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B82A14606B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:31:59.598038789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:31:59.961145899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:31:59.967918453Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B82D54120F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:32:00.072586336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:32:00.077377961Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B82DBC69B96, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:32:01.455651168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:32:01.457928971Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B832E1096FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:32:01.457950783Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:32:01.457971293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:32:01.459845806Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B832E3077A6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:32:04.010536569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:32:04.013829509Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B83C65D57E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T03:32:04.597659028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:32:09.598002754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:32:13.035215931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:32:13.037985348Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B85E04909C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:32:14.597702947Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:32:14.597702895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:32:14.597731768Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:32:14.602420445Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-07-02T03:32:14.604574241Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006945674s
level=info ts=2026-07-02T03:32:15.191974619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:32:15.198097761Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B8661086EC7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:32:15.829841281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:32:15.832192386Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B8686D63DB1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:32:16.026771709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:32:16.029173844Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B8692941462, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:32:16.265751736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:32:16.27020715Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B86A0F2F30D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:32:16.948773152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:32:16.953437076Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B86C9ABB59A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=warn ts=2026-07-02T03:32:19.598118266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:32:19.973969383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:32:19.97631345Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B877DD867EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:32:20.61042748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:32:20.612719976Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B87A3C6F6D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:32:21.436842588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:32:21.44113592Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B87D527287E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:32:23.686682822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:32:23.69168021Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B885B4BEA86, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:32:23.834048464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:32:23.836346237Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B8863EC492D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T03:32:24.597517024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:32:24.885270658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:32:24.887641149Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B88A295426D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:32:24.957733607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:32:24.96010324Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B88A6E62A35, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:32:25.849779482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:32:25.852118114Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B88DC126F52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:32:28.300024787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:32:28.302384113Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B896E1D84B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:32:28.939091188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:32:28.941509408Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B89943646C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:32:29.598168828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:32:31.080198709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:32:31.08253237Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B8A13D47090, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:32:31.382379019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:32:31.384717476Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B8A25D6A355, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:32:31.72662876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:32:31.729335723Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B8A3A618990, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-02T03:32:34.597806022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:32:38.411850987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:32:38.418194639Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B8BC911B124, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:32:38.418213872Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:32:38.418234338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:32:38.420271624Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B8BC9335C42, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T03:32:39.597830649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:32:44.579074979Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:32:44.597188795Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T03:32:44.597218416Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:32:44.597245581Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:32:44.597307325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:32:44.597317776Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-07-02T03:32:44.597325832Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T03:32:44.597342845Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.577µs success=true
level=info ts=2026-07-02T03:32:44.627442918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:32:44.634147916Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B8D3B91ADC7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:32:44.634166349Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:32:44.634192398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:32:44.642945789Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B8D3C18C74D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:32:44.820062968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:32:44.825221954Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B8D46F4BD6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:32:45.811094453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:32:45.816022153Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B8D820267AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:32:46.41785812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:32:46.420092714Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B8DA604F6A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:32:46.698951512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:32:46.702195886Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B8DB6C92A7E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T03:32:49.598210535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:32:49.755532902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:32:49.75779053Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B8E6CF649CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:32:50.560945325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:32:50.563469049Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B8E9CFB1E95, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:32:53.036351483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:32:53.03875284Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B8F30854F3A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T03:32:54.59792787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:32:57.489278008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:32:57.491626529Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9039EE4FBF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:32:59.089217617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:32:59.096216312Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B90999269C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:32:59.431871761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:32:59.43862994Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B90ADF8B252, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-02T03:32:59.597449955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:32:59.819447281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:32:59.823827304Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B90C4F17693, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:32:59.823847186Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:32:59.823869699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:32:59.830347816Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B90C555EED8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:33:02.62644266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:33:02.631081176Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B916C43E9B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:33:04.597300818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:33:05.910238721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:33:05.912546618Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B922FDBF3AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:33:07.682441494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:33:07.685258351Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9299857B90, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:33:09.224875348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:33:09.230057704Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B92F595D6E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T03:33:09.598028131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:33:11.094597373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:33:11.096971649Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9364DF3D42, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:33:11.230386204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:33:11.232708108Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B936CF7983C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:33:11.307938186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:33:11.310344082Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B937197CC6E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:33:11.310363648Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:33:11.310382923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:33:11.312163641Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9371B5F8DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:33:12.271355474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:33:12.273692048Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B93AB024F7C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-07-02T03:33:12.27371011Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:33:12.273730979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:33:12.275773085Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B93AB22D10F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:33:13.785476306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:33:13.787878605Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9405430140, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:33:14.597675046Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:33:14.597701061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:33:14.597714067Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:33:16.300703706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:33:16.303072143Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B949B2E19AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:33:16.463604979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:33:16.467799925Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B94A4FFE6B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:33:16.507929399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:33:16.514672844Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B94A7CB0DC2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:33:16.991070219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:33:16.99569999Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B94C477F624, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T03:33:19.597375921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:33:19.838905299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:33:19.8412306Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B956E134721, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:33:21.206444548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:33:21.208914619Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B95BF95FC0F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:33:21.637990141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:33:21.640301704Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B95D94EBC01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:33:22.037533432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:33:22.040187137Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B95F125249F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:33:23.398070876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:33:23.402861871Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B96425D21CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:33:24.184782081Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE5B9670EE61C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T03:33:24.597525224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:33:26.364065317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:33:26.366430186Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B96F3009147, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:33:27.228049479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:33:27.23050734Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9726817DE6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T03:33:29.597993992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:33:31.141232351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:33:31.144736727Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B980FC32571, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:33:31.144755936Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:33:31.144779514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:33:31.147215804Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B980FF3479E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:33:32.200604807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:33:32.202978828Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B984EE398E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T03:33:34.597389622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:33:35.900916492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:33:35.903719161Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B992B7861BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:33:37.731264456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:33:37.737468543Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9998C5BA25, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:33:38.455437974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="96 MB" avg_uncomp="32 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:33:38.462178942Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B99C3F8B421, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:33:38.610917737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:33:38.615712131Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B99CD1EA2C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:33:39.598121408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:33:41.50052249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:33:41.502841592Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9A79346E7D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:33:42.974950065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:33:42.97734254Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9AD1175C6B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:33:42.977358609Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:33:42.977378036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:33:42.97919971Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9AD13675FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:33:44.386032915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:33:44.388541191Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9B25347327, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:33:44.578709492Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:33:44.597821075Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:33:44.597853904Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:33:44.597843875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:33:45.124259337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:33:45.128696197Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9B51524C5F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:33:48.311562115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:33:48.317949399Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9C0F5DB95C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:33:49.141371412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:33:49.144190185Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9C40AAF9EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-07-02T03:33:49.597660588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:33:50.63776437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:33:50.642370984Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9C99F6D532, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:33:52.5027244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:33:52.505055832Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9D08FD4364, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:33:53.991013563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:33:53.993363426Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9D61B2BEF6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:33:54.598074194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:33:54.944649928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:33:54.947037552Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9D9A8A2750, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:33:54.947058456Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:33:54.947079502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:33:54.948927559Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9D9AAA9B69, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:33:55.17878694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:33:55.181516302Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9DA8810AD0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:33:55.181531694Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:33:55.247746815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:33:55.250762609Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9DAC9F86CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:33:55.279909705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:33:55.282138838Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9DAE84CA84, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:33:56.500592039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:33:56.502908617Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9DF74780E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:33:56.502925502Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:33:56.50294589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:33:56.504732197Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9DF76624F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-02T03:33:59.597923292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:34:02.585327964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:34:02.587734922Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9F61F65014, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:34:02.587756787Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:34:02.587775962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:34:02.589582056Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5B9F62152B9C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T03:34:04.597431517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:34:05.310375694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:34:05.312752074Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA00462C3BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:34:05.312767852Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:34:05.312786604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:34:05.314628194Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA004817E5F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:34:06.203241498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:34:06.205560875Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA0399A6536, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:34:06.526291604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:34:06.528711033Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA04CDCE2B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:34:06.528728494Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:34:06.528747809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:34:06.530676056Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA04CFD5218, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:34:07.426934142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:34:07.429206348Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA0828A4A4C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:34:07.429224482Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:34:07.42924585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:34:07.431102079Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA082A9AA8A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:34:07.879545924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:34:07.881862929Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA09D849DD0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:34:07.881879394Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:34:07.881896265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:34:07.883625735Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA09DA2317C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-07-02T03:34:09.597631765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:34:10.90469987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:34:10.908843759Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA151F054E7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:34:11.639938651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:34:11.646251482Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA17DE1D1F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:34:12.232845222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:34:12.235312248Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA1A0FFB132, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:34:12.966447158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:34:12.968777123Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA1CCB8B122, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:34:14.597783671Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:34:14.597813597Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:34:14.597828113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:34:16.345698836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:34:16.34801137Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA296236A7B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:34:16.348030128Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:34:16.348048453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:34:16.349906785Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA2964303AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:34:16.649873207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:34:16.652672416Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA2A84BF383, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:34:18.064213625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:34:18.069184828Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA2FCBA890B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:34:18.169291591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:34:18.171528636Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA302D4E7EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:34:19.013214801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:34:19.015725597Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA335247D4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T03:34:19.59828885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:34:19.921695155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:34:19.924254178Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA36B4C9FD4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:34:20.571978406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:34:20.574423352Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA3920DAE50, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:34:20.574440708Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:34:20.574460014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:34:20.576320004Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA3922CB2F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:34:21.348314528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:34:21.351371638Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA3C05685B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:34:21.351390673Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:34:21.351428613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:34:21.353743927Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA3C07F8914, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:34:22.241083269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:34:22.243531679Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA3F5887BF0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:34:22.955054661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:34:22.957311589Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA42015DEDC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:34:23.169716901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:34:23.17316985Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA42CE6676D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:34:24.267157418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:34:24.269527781Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA46E4BD007, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:34:24.597299655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:34:26.446924103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:34:26.453785158Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA4F07D825E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:34:26.998645969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:34:27.003360861Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA5113EF5BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:34:29.487737866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:34:29.490556178Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA5A57EF3C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:34:29.490574945Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:34:29.490597784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:34:29.495501958Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA5A5CBD990, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-07-02T03:34:29.597430055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:34:31.155387066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:34:31.160207143Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA60901FF4B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:34:31.68865938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:34:31.691006397Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA628A741FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:34:31.823446345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:34:31.825824768Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA630B052A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:34:33.352687342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:34:33.354982023Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA68BD61666, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:34:34.112251416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:34:34.115203649Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA6B924C5A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-07-02T03:34:34.597911508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:34:34.725271933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:34:34.732174899Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA6DDEC0644, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:34:34.770552737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:34:34.775018445Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA6E0797DF1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:34:35.464420139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:34:35.466714474Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA709B34BEB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:34:35.823902841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:34:35.826309142Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA71F226226, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:34:39.456443594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:34:39.458740647Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA7F7A51057, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-07-02T03:34:39.597515518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:34:43.016665927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:34:43.019319985Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA8CBDD43DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:34:43.237868246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:34:43.240112302Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA8D909548F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:34:44.578541212Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:34:44.597659307Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:34:44.597685967Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:34:44.597705655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:34:45.46077198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:34:45.463202973Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA95D89BABD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:34:45.688020078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="513 B" avg_comp="513 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:34:45.690279676Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA96B141629, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:34:45.750370691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:34:45.753119143Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BA96ED2B772, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:34:49.430062151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:34:49.435656302Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BAA4A50FE41, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T03:34:49.597277686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:34:51.774446089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:34:51.776768469Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BAAD5DBE37F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:34:52.269640348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:34:52.272015973Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BAAF360C246, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:34:53.867159878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:34:53.871491743Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BAB52B59E65, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T03:34:54.597344945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:34:56.68536759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:34:56.69002264Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BABFAB55DED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:34:58.85309831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:34:58.855135483Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BAC7BC59A62, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T03:34:59.597793787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:35:04.598217083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:35:06.116910376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:35:06.120381956Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BAE2CC0D298, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:35:06.290855799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:35:06.293204444Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BAE371B3DB1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:35:07.748628381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:35:07.75107804Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BAE8DFF6ED1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:35:07.751095564Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:35:07.751116192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:35:07.752992641Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BAE8E2045EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T03:35:09.597951049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:35:10.05919265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:35:10.061457233Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BAF17B5ABD9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:35:10.061473651Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:35:10.061494272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:35:10.063365755Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BAF17D4592A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:35:10.870859922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:35:10.873249395Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BAF4819CAF1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:35:12.330625869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:35:12.33320042Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BAF9F1CD483, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T03:35:14.597525838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:35:14.597521455Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:35:14.597555194Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:35:14.664863343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:35:14.667558114Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB02A41B099, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:35:15.755280161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:35:15.760187307Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB06B620804, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:35:15.760207167Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:35:15.760228551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:35:15.762062948Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB06B80B574, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:35:16.831760358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:35:16.838795015Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB0ABA956DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:35:17.679599805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:35:17.68402358Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB0DE0DB269, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=warn ts=2026-07-02T03:35:19.598292674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:35:19.634460012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:35:19.63683016Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB152721203, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:35:19.655864151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:35:19.658176133Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB153B955B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:35:19.873908083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:35:19.876794271Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB160BFB738, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:35:20.407011467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:35:20.411755342Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB180A2E984, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:35:22.059880008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:35:22.062203749Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB1E302F84F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:35:22.843705855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:35:22.846398301Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB211C16E38, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-07-02T03:35:24.597890691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:35:24.876928079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:35:24.881897666Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB28B1069BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:35:27.166431998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:35:27.16900111Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB3136423C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:35:28.424603569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:35:28.42688879Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB35E5F80AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:35:29.597900645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:35:29.600032013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:35:29.602366301Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB3A46FDEA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:35:30.466823843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:35:30.46914641Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB3D81A630A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:35:32.470593346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:35:32.472998812Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB44F8A12DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:35:33.088704833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:35:33.0909413Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB4745FE040, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T03:35:34.598196901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:35:34.684665374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:35:34.69022805Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB4D3B2F7E7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:35:35.634025102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:35:35.640574123Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB50C586055, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:35:37.242454014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:35:37.246787981Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB56C145F0E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:35:37.246817381Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:35:37.246843202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:35:37.253227673Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB56C78BD2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:35:37.342456359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:35:37.347106188Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB5720FA276, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T03:35:39.598154679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:35:42.886275387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="663 B" avg_comp="663 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:35:42.888715021Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB6BC5D6BE8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:35:43.817349868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:35:43.81970971Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB6F3DBA4DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:35:44.578437681Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T03:35:44.597964258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:35:44.597974479Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:35:44.598011597Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:35:45.03204307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:35:45.035054126Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB73C436C02, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:35:48.77602805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:35:48.778503412Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB81B6BF584, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T03:35:49.598083799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:35:50.197085483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:35:50.201345612Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB8703BCDE1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:35:51.084272382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:35:51.089718313Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB8A52F0617, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:35:52.896457143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:35:52.899213478Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB911094C71, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:35:53.382797308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:35:53.385183207Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB92E008EF4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:35:54.597749268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:35:55.82137637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:35:55.823776594Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB9BF5AA681, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:35:56.647172967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:35:56.649562732Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BB9F0928D1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:35:59.350114802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:35:59.352454632Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BBA91AFA98D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=warn ts=2026-07-02T03:35:59.598216618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:36:01.257623944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:36:01.26046555Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BBB0367CC53, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:36:03.452519959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:36:03.458315588Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BBB865D78C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T03:36:04.597263334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:36:05.492747163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:36:05.495657336Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BBBFFD83095, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:36:06.998013481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:36:07.002869964Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BBC59AD23B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-07-02T03:36:09.597461539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:36:13.591371151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:36:13.594534464Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BBDE2933ACA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:36:14.24998094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:36:14.255065093Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BBE09F1ECFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:36:14.597226051Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:36:14.597262993Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:36:14.597278184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:36:15.874015944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:36:15.876428719Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BBE6A955607, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:36:15.885444484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:36:15.89185431Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BBE6B82B35A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:36:17.196069609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:36:17.200839877Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BBEB987361A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:36:18.5056239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:36:18.510119807Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BBF07924A0F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T03:36:19.598032984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:36:21.23835469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:36:21.243737781Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BBFAA7F1D3C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:36:21.644089897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:36:21.646470268Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BBFC2825115, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:36:22.948285422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:36:22.950599756Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BC0103EBA9D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:36:23.066904943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:36:23.069136017Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BC0174FCBB9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:36:24.507169122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:36:24.509954363Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BC06D2CD52E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-02T03:36:24.597433364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:36:25.441759611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:36:25.445986321Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BC0A4F9AE81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:36:26.600517805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:36:26.605298113Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BC0EA135A11, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T03:36:29.598214643Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:36:29.967532665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:36:29.969886007Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BC1B2A0B2F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:36:31.822167497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:36:31.824759566Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BC2212D6E2D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:36:31.89295048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:36:31.895234995Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BC225628878, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:36:33.783821841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:36:33.786192487Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BC296189810, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:36:34.557185811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:36:34.559539011Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BC2C42FFE83, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T03:36:34.597524672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:36:39.597325284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:36:40.139636403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:36:40.141891639Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BC410ECF1D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:36:42.931720624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:36:42.934011909Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BC4B75A0324, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:36:43.732178898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:36:43.734835981Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BC4E713E08F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:36:44.579012537Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:36:44.597128479Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:36:44.59715755Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:36:44.598236925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:36:47.665929703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:36:47.670395691Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BC5D1A8F6BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:36:49.002158267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:36:49.004389513Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BC6212C38D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:36:49.598076832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:36:50.873745564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:36:50.875877636Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BC690B90C31, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:36:52.849795156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:36:52.853201141Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BC70687C765, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:36:53.999822941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:36:54.002062654Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BC74B0F1615, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-07-02T03:36:54.598150062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:36:58.538758481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:36:58.541083406Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BC85999BC3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:36:59.033451238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:36:59.036228683Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BC8771D553F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-07-02T03:36:59.59773782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:37:04.597285631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:37:05.769822742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:37:05.775960858Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCA08D473B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:37:07.531053817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:37:07.533307808Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCA7194C1FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:37:08.407456764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:37:08.410059772Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCAA5D4C12C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:37:08.588710248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:37:08.591094285Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCAB0A1B87C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:37:08.648149497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:37:08.650768224Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCAB42D14FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-02T03:37:09.598143653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:37:09.824317513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:37:09.82709242Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCAFA4CE40C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:37:11.323010028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:37:11.332577592Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCB54080FBB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:37:11.336597944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:37:11.340562194Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCB5483A2BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:37:13.079702098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:37:13.0842216Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCBBC7094CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:37:13.788317241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:37:13.793081905Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCBE6B14921, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:37:13.79310105Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:37:13.793123431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:37:13.795272709Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCBE6D4357F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:37:14.597780313Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:37:14.597806943Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:37:14.597832226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:37:14.603289243Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-07-02T03:37:14.605429301Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007952424s
level=info ts=2026-07-02T03:37:18.647513425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:37:18.649928914Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCD082E65B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:37:18.969796528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:37:18.972254754Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCD1B63F6EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T03:37:19.597465407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:37:19.705722427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:37:19.708015739Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCD473F1508, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:37:20.426325717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:37:20.42863447Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCD7232FC83, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:37:21.272078766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:37:21.276197711Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCDA4B86A37, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:37:21.498764106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:37:21.50370981Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCDB246BFBA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=warn ts=2026-07-02T03:37:24.597958088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:37:25.928249164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:37:25.930578459Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCEBA2457F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:37:27.280730578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:37:27.283077772Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCF0AC268DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:37:28.1362572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:37:28.138931904Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCF3DC587C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-07-02T03:37:28.138952279Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:37:28.138979633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:37:28.144218934Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCF3E17298B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-02T03:37:29.598008252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:37:30.33556208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:37:30.340823509Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BCFC0FFBA97, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T03:37:34.598309589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:37:35.306418046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="920 B" avg_comp="920 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:37:35.308936112Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD0E921C602, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:37:35.796447895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:37:35.79988125Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD1065A4D4B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:37:38.1045921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:37:38.106893498Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD18FE87CB6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:37:39.192852371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:37:39.195227956Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD1D0C6E7B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:37:39.597294032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:37:41.24395589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:37:41.246248384Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD24B0789CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:37:44.578567169Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:37:44.597854243Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:37:44.597863108Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T03:37:44.597880243Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:37:44.597942352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:37:44.59810792Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-07-02T03:37:44.598117385Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T03:37:44.598135795Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.974µs success=true
level=info ts=2026-07-02T03:37:46.105077822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:37:46.107988556Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD36CCDE864, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-07-02T03:37:47.566077848Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE5BD3C3BB873F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T03:37:47.575334466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:37:47.579383261Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD3C48487A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:37:49.597808199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:37:49.933040411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:37:49.935694443Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD450F3CBA0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:37:52.229806413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:37:52.23275984Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD4D9DEE44A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:37:52.232777806Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:37:52.232798376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:37:52.237466171Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD4DA292E55, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:37:53.176418744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:37:53.181297016Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD51269AF05, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:37:53.439897863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:37:53.442256833Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD521F6F486, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-07-02T03:37:54.597719659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:37:54.934001203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:37:54.936835039Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD57B0CE5BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:37:54.987169007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:37:54.991650968Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD57E50E9F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:37:58.468955446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:37:58.471281889Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD64DB8CB7A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:37:58.805368262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:37:58.812479309Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD6620E85C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T03:37:59.598246647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:37:59.673569915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:37:59.678806542Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD695AF77C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:38:01.025459245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:38:01.030173204Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD6E63C4718, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:38:02.1412123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:38:02.146102984Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD728C03EDF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:38:02.555142639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:38:02.55930473Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD74161931C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T03:38:04.598090953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:38:05.85576408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:38:05.86095508Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD8062C52E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T03:38:09.597800712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:38:09.619893782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:38:09.622252524Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD8E65F6002, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:38:10.672599421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:38:10.675393981Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD925249D28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:38:10.794757303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:38:10.799584821Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD92C8B763C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:38:12.521030012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:38:12.525988313Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD99370A55F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:38:13.349559828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:38:13.351866164Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD9C4ACAE80, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:38:13.351881565Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:38:13.351901273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:38:13.353678232Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD9C4CA3440, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:38:13.665827058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:38:13.66801481Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BD9D7851925, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:38:14.597026596Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:38:14.597058057Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:38:14.59811705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:38:15.00415283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:38:15.006518854Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BDA274B9FFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:38:16.942107456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:38:16.944334855Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BDA9ACD1A53, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:38:18.870983816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:38:18.873399729Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BDB0DC809F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T03:38:19.597997549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:38:19.940097051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:38:19.942590727Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BDB4D81B20E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:38:21.944538704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:38:21.946976146Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BDBC4F9A707, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:38:23.419705851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:38:23.422122089Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BDC1CE75C43, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=warn ts=2026-07-02T03:38:24.597307558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:38:26.757237666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:38:26.760578804Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BDCE3D80EAB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:38:28.936761233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:38:28.939114605Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BDD65BEB16C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:38:29.598046189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:38:32.683043167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:38:32.68582975Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BDE450F8D38, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-07-02T03:38:34.598227644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:38:35.467791224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:38:35.473846586Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BDEEB3E9973, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:38:36.288077687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:38:36.29033212Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BDF1BE99B84, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:38:36.538893487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:38:36.541240833Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BDF2ADE331C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:38:39.083728359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:38:39.086547742Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BDFC29415DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-07-02T03:38:39.597642119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:38:42.867837689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:38:42.872760805Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE0A4402BCC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:38:43.78542189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:38:43.794375418Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE0DB2F1BAA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:38:44.075015288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:38:44.079558035Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE0EC2EDBA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:38:44.578993557Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:38:44.597318224Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:38:44.597347148Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:38:44.597348691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:38:45.190238866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:38:45.192678733Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE12E8709C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:38:45.267766963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:38:45.270007044Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE13324A9E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:38:46.982393677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:38:46.98721698Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE1997F4D59, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:38:47.552638014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:38:47.557214953Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE1BB77B202, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:38:48.25811609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:38:48.260883949Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE1E565CED4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:38:48.402511173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:38:48.404813525Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE1EDFD55D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T03:38:49.597418511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:38:50.068159632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:38:50.070486277Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE25146524F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:38:51.156139318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:38:51.158899132Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE292255BC1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:38:51.484007585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:38:51.488809878Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE2A5CEACE8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:38:52.138603071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:38:52.142975704Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE2CCCC3967, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:38:53.112455024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:38:53.117361242Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE306E0C2AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T03:38:54.597282752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:38:56.706227738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:38:56.708790873Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE3DCF13640, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T03:38:59.598074565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:39:04.564836648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:39:04.567572641Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE5B15B3D30, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:39:04.567593864Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:39:04.567612592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:39:04.569443255Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE5B17C0636, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T03:39:04.597930621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:39:06.383011609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:39:06.385295154Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE61DB63FE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:39:07.174553422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:39:07.176922266Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE64CE4869B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T03:39:09.597452983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:39:10.364967229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:39:10.367321587Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE70B0DD856, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:39:14.597690614Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:39:14.597716397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:39:14.597726814Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:39:14.972608002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:39:14.975079565Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE81DB30403, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T03:39:19.598210197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:39:20.448876048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:39:20.451236432Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE9641B5D74, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:39:22.051222806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="513 B" avg_comp="513 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:39:22.053602709Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BE9C39CFCD3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:39:23.319959621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:39:23.323501864Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BEA0F40C45C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T03:39:24.597264414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:39:26.658739687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:39:26.661334739Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BEAD6409811, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:39:26.978372482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:39:26.981181294Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BEAE95249AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:39:27.127440843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:39:27.132773901Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BEAF25B33CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:39:27.607433671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:39:27.616481343Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BEB0F2F3E7D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:39:27.61649964Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:39:27.616522243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:39:27.625532459Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BEB0FB9DB45, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:39:28.353325746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:39:28.360097539Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BEB3B80B571, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:39:28.36012353Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:39:28.360155509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:39:28.366883345Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BEB3BE8F3B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:39:29.104135997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:39:29.108753152Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BEB68225899, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:39:29.597728474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:39:30.206668201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:39:30.208966881Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BEBA9B693EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:39:34.597715064Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:39:35.575774471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:39:35.578214018Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BECE9BDFB9A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:39:37.676156958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:39:37.67862797Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BED66EF3B7E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:39:37.684663268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:39:37.686605675Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BED676B79E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:39:37.686619304Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:39:37.686638848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:39:37.68829636Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BED678716E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:39:38.46639967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:39:38.468765302Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BED96087A5D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-07-02T03:39:38.468782566Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:39:38.468801221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:39:38.470502273Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BED9625FF6B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:39:38.969459882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:39:38.971826923Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BEDB404C26B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:39:39.537277157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:39:39.539650426Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BEDD5DD33F0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:39:39.598073204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:39:40.184442267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:39:40.186693286Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BEDFC6EA094, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:39:41.834889107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:39:41.837327392Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BEE5ED05B24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:39:42.423324652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:39:42.425699331Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BEE81E1DC64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:39:42.517920524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:39:42.522673382Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BEE87AAE3B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:39:44.579005819Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T03:39:44.59757467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:39:44.5975836Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:39:44.59761175Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:39:46.143432904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:39:46.154514513Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BEF602305B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:39:46.844336804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:39:46.84966796Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BEF89927D54, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:39:48.230458623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:39:48.235438889Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BEFDC2A10B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:39:49.125849025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:39:49.128150451Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF01161C65B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T03:39:49.598231394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:39:50.571172652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:39:50.575717765Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF067A96C28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:39:51.373914475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:39:51.378725389Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF0978696E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:39:53.786588036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:39:53.788904464Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF12730580A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T03:39:54.597385074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:39:56.429380573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:39:56.431997305Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF1C4B7A6DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:39:59.432010965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:39:59.43444969Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF277AF1BBA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:39:59.43446676Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:39:59.434486536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:39:59.436253469Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF277CD59DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-07-02T03:39:59.59809977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:40:00.191836468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:40:00.194196925Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF2A4F7F89C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:40:00.975503397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:40:00.977892762Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF2D3AD953C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:40:03.884451917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:40:03.890439996Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF381486AAD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T03:40:04.597555432Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:40:08.964052528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:40:08.970559301Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF4B014EF87, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:40:09.598001282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:40:10.40858897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:40:10.410998673Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF505F0611E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:40:10.510216033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:40:10.517081279Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF50C437C57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:40:10.600530369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:40:10.605742122Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF5118C0298, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:40:13.482661818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:40:13.487613354Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF5BD506F2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:40:14.597193848Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:40:14.597222521Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:40:14.597260036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:40:14.738832709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:40:14.741695234Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF60811CB04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:40:18.054899827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:40:18.059994165Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF6CDDAAC01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:40:18.79006424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:40:18.792349655Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF6F9823BA8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:40:19.322869176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:40:19.325260575Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF71945FD8F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:40:19.325277518Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:40:19.325297371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:40:19.327040951Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF71963C2EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-02T03:40:19.597860616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:40:20.156568687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:40:20.159004625Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF74AF69B50, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:40:21.106589085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:40:21.108958731Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF783965DA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:40:21.108973794Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:40:21.10899358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:40:21.110794835Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF783B519C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:40:21.400753544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:40:21.404331196Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF795222F52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:40:21.40434822Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:40:21.409496559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:40:21.411885443Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF795A2AD6E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:40:21.894269698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:40:21.896556957Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF7B287CFE8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:40:24.598145973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:40:29.361663509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:40:29.364584617Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF96FA4C0F8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:40:29.364600505Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:40:29.364619847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:40:29.366915146Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BF96FCA0DE3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-02T03:40:29.598304479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:40:32.99673769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:40:33.000288902Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFA4850DE5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:40:34.116267605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:40:34.118668277Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFA8B058220, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:40:34.239033722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:40:34.241358506Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFA92570892, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:40:34.241375975Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:40:34.241394283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:40:34.243196921Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFA927634B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-07-02T03:40:34.598173751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:40:34.822760995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:40:34.825155301Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFAB5231288, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:40:36.13470655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:40:36.136989976Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFB0353CA60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:40:36.137008749Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:40:36.137031771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:40:36.138814134Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFB03723112, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:40:36.74063641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:40:36.743025298Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFB2772D490, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:40:36.743047453Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:40:36.743073849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:40:36.745049653Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFB2794F2E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:40:37.325368972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:40:37.328165791Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFB4A53C8C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:40:37.723054282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:40:37.728737356Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFB6233C211, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:40:37.843074676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:40:37.845458441Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFB6929096B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:40:38.103087631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:40:38.107262849Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFB78C44874, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:40:38.142428982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:40:38.14718926Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFB7B249823, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:40:38.536254693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:40:38.538690514Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFB9279EF49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:40:38.949760152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:40:38.952152814Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFBAB1FB28A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:40:38.952168805Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:40:38.952199159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:40:38.954059656Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFBAB3FF749, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:40:39.48861675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="663 B" avg_comp="663 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:40:39.49084276Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFBCB3C6C65, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:40:39.490860721Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:40:39.490879182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:40:39.492689281Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFBCB5A1F51, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:40:39.598054759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:40:44.57847824Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T03:40:44.597562446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:40:44.597568522Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:40:44.597601089Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:40:45.717197582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:40:45.720021979Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFD3E829CDB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:40:45.818366728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:40:45.820739754Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFD4486B484, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:40:47.134772495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:40:47.137238265Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFD92FE1BE5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:40:47.712109183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:40:47.714435347Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFDB566AA29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:40:49.406289832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:40:49.410908341Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFE1A8377D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T03:40:49.597591375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:40:50.310729158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:40:50.315517465Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFE506EFD96, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:40:53.187141107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:40:53.189496068Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFEFBBBA0F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:40:53.18951438Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:40:53.18953181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:40:53.191325262Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFEFBDB12AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:40:53.78804147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:40:53.790375351Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFF1F8DB78F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:40:53.790391075Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:40:53.790426558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:40:53.792197503Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFF1FABACD2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:40:54.193608779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:40:54.195991823Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFF37BA5A5D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:40:54.597928128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:40:55.34960837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:40:55.35185775Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFF7CA02246, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:40:56.00618822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:40:56.012998495Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFFA4080297, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:40:56.089193111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:40:56.094162653Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFFA8DD849C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:40:56.094182867Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:40:56.094206878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:40:56.096085159Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFFA8FDD2E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:40:57.448852277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:40:57.451664883Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5BFFF9C7BEF2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-07-02T03:40:59.597943397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:41:03.432730029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:41:03.437568799Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C015E917FEB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:41:04.086439535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:41:04.088856146Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0185630BEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T03:41:04.597535692Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:41:05.22164953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:41:05.224081114Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C01C90CBCEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:41:05.942951668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:41:05.945478578Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C01F40CE44E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:41:06.764702004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:41:06.768176041Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C022509D7E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:41:07.341857628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:41:07.344712827Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C024773CD88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:41:07.3447312Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:41:07.344753242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:41:07.349380228Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0247BD8FAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:41:08.013570122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:41:08.018480622Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C026F9CD14E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T03:41:09.598173477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:41:13.807045114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:41:13.809557534Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C03C8C9611C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:41:14.47023916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:41:14.472657652Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C03F04F11C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:41:14.596968956Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:41:14.596998622Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:41:14.598054721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:41:16.34729484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:41:16.349576984Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C04602F74DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:41:17.278285764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:41:17.280672884Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0497AE78C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:41:18.253377578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:41:18.255799528Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C04D1CE1D5F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:41:19.598141938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:41:20.490389022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:41:20.494703831Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0557407604, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:41:20.565709388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:41:20.572415082Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C055BE22D13, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:41:21.094001109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:41:21.098512905Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C057B3EF44A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:41:21.806550093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:41:21.808938764Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C05A595B9CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:41:24.597578032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:41:25.386718781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:41:25.389038689Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C067AFB54E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:41:25.634758677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:41:25.637383963Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0689C5529E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:41:26.833893195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:41:26.836849124Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C06D14600DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:41:27.58443851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:41:27.589376808Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C06FE203782, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:41:27.927114718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:41:27.929353365Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0712652BB5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=warn ts=2026-07-02T03:41:29.597898529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:41:32.043390239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:41:32.045729854Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0807BFB338, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:41:32.984310043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:41:32.986692425Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C083FD44F45, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:41:33.234959596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:41:33.237365974Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C084EC6513D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:41:34.002596088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:41:34.004969849Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C087C854439, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:41:34.19057481Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE5C0887991873, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T03:41:34.597911118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:41:37.54272426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:41:37.545177125Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C094F8A377E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T03:41:39.59811127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:41:41.509676384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:41:41.515635692Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0A3C326B05, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:41:41.724538427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:41:41.730211795Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0A48FC35FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:41:43.505618212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:41:43.510466501Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0AB319970D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:41:43.510487513Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:41:43.510508272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:41:43.512278986Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0AB337D7D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:41:43.513273917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:41:43.514903339Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0AB360FEC7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:41:44.578319085Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:41:44.597295675Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:41:44.597323635Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:41:44.597327216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:41:45.020366133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:41:45.028221875Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0B0D914138, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:41:47.244717542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:41:47.250907031Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0B920AB554, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-07-02T03:41:49.598311312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:41:50.71526774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:41:50.720885863Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0C60D3FA26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:41:50.882455945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:41:50.884795922Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0C6AA47C31, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:41:50.884817389Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:41:50.884837885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:41:50.886681687Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0C6AC3CF29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:41:51.837295187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:41:51.839641074Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0CA38EE00B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T03:41:54.598238699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:41:57.849616344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:41:57.852004574Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0E09EB4F38, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:41:57.852021146Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:41:57.852040255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:41:57.854003028Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0E0A0CC452, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:41:58.260112032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:41:58.262463366Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0E226373A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-02T03:41:59.597401242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:42:00.008774415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:42:00.011090091Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0E8A9D93D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:42:03.041087712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="920 B" avg_comp="920 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:42:03.043499203Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0F3F5B49A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:42:04.412556816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:42:04.415363094Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0F9120F315, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-02T03:42:04.597305838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:42:05.381890183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:42:05.387802634Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0FCB1769D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:42:05.638516115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:42:05.642669287Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0FDA48A717, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:42:05.642688734Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:42:05.642712447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:42:05.648842558Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C0FDAA859B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:42:08.777604184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:42:08.782440545Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C10956CC770, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:42:09.319684199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:42:09.321968797Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C10B595BB80, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=warn ts=2026-07-02T03:42:09.597735194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:42:09.808331852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:42:09.81051915Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C10D2B55842, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:42:10.868728419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:42:10.870993179Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C1111EA76F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:42:11.272057626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:42:11.274346093Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C1129F4C613, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:42:14.566689927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:42:14.569234757Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C11EE58CDB5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:42:14.597564981Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:42:14.597571401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:42:14.597594261Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:42:14.603680432Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-07-02T03:42:14.60587888Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007606763s
level=info ts=2026-07-02T03:42:16.245753034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:42:16.248172785Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C12526A9FBC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:42:16.618363493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:42:16.620739128Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C12689F9550, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:42:18.375655227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:42:18.379893499Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C12D17A5416, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:42:19.270522112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:42:19.275892238Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C1306DF3FD5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-02T03:42:19.597485189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:42:24.597325417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:42:25.289430466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:42:25.292289219Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C146D7D28F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:42:27.449701073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:42:27.454773483Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C14EE628774, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T03:42:29.598020737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:42:30.232994677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:42:30.235481298Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C15941FD435, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:42:31.438437495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:42:31.440920082Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C15DBF9613F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:42:32.740757509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:42:32.744818058Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C1629B13563, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T03:42:34.597317648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:42:35.018069095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:42:35.020438015Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C16B154DA3A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:42:38.647836865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:42:38.651362068Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C1789B215B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:42:38.924560528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:42:38.929186359Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C179A5143C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:42:39.230033624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:42:39.241395184Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C17ACEB5536, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T03:42:39.598126345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:42:40.455335507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:42:40.459961076Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C17F58D18E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:42:40.713815395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:42:40.716142577Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C1804D3FD81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:42:40.720180135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:42:40.722655688Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C18053776EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:42:42.15152363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:42:42.156102099Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C185AA767A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:42:42.959395155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:42:42.961688215Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C188AAAB511, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:42:44.578348175Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:42:44.597858607Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T03:42:44.597880354Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:42:44.59790911Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:42:44.597929061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:42:44.59798987Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-07-02T03:42:44.597997193Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T03:42:44.598014719Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.014µs success=true
level=info ts=2026-07-02T03:42:45.278279891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:42:45.280793857Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C1914E38057, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:42:46.224605348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:42:46.226994746Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C194D4B63C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T03:42:49.597534403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:42:52.406891654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:42:52.409161944Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C1ABDC86B56, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:42:53.92444767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:42:53.929022174Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C1B185F0485, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T03:42:54.598055856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:42:57.615445943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:42:57.621274442Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C1BF4723613, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:42:58.439379404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:42:58.441705908Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C1C255924BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=warn ts=2026-07-02T03:42:59.598023843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:42:59.603107213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:42:59.60530746Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C1C6AB56055, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:42:59.967062985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:42:59.969825694Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C1C806BAC59, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:43:01.67302359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:43:01.675693782Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C1CE61C852D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:43:01.703731033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:43:01.708461818Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C1CE81135AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:43:03.852781013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:43:03.855196322Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C1D6805116A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-07-02T03:43:04.598230554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:43:06.441049325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:43:06.443769626Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C1E024F859A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:43:08.758194959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:43:08.762919415Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C1E8C8A2B4E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T03:43:09.598043449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:43:11.236457115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:43:11.238879455Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C1F201DCC3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:43:14.597113103Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:43:14.5971432Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:43:14.598215156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:43:15.332735476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:43:15.33506604Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2014464286, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T03:43:19.597851279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:43:20.846717596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:43:20.853484941Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C215D328A3D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:43:22.35927949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:43:22.364188216Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C21B73CE168, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:43:23.538452721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:43:23.541711138Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C21FD6196DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:43:23.977223031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:43:23.979539926Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2217867BD4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:43:24.148752817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:43:24.151586151Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2221C7449D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:43:24.558716004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:43:24.563346475Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C223A524743, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T03:43:24.597287958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:43:26.11820429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:43:26.120660376Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C22972434F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:43:26.120678394Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:43:26.120697562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:43:26.122593252Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C229744AFB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:43:26.492602573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:43:26.495001466Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C22AD749536, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:43:27.024636839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:43:27.026916082Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C22CD291C83, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:43:28.853356743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:43:28.85596422Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C233A2CAD5D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T03:43:29.597456296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:43:31.369988114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="938 B" avg_comp="938 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:43:31.372365073Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C23D02B896C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:43:32.124547769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:43:32.129010021Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C23FD441344, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T03:43:34.597999214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:43:35.951524364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:43:35.958076047Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C24E17FFD22, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:43:37.204990884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:43:37.207369559Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C252BF68B2B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:43:38.51464296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:43:38.517052939Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C257A05BCAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:43:38.728244205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:43:38.730652729Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2586C1761E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:43:39.597465335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:43:40.456458153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:43:40.458924356Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C25EDC400BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:43:40.804035213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:43:40.808652338Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C260298FB22, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:43:44.578462093Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:43:44.597199664Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:43:44.597226832Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:43:44.59724746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:43:44.711347478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:43:44.716330263Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C26EB883CB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:43:44.771181635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:43:44.773799948Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C26EEF4735A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-07-02T03:43:44.773815485Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:43:44.773835437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:43:44.778267771Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C26EF3A7ABC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:43:47.666992395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:43:47.671750917Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C279BAE1D38, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-07-02T03:43:49.597875164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:43:54.597733464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:43:55.582085138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:43:55.584629264Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C297354555E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:43:55.871507162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:43:55.874329507Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C298498814C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:43:55.874347705Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:43:55.874367043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:43:55.879264349Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2984E7B959, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:43:56.601513477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:43:56.60607936Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C29B0374C61, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:43:58.554706721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:43:58.55697864Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2A247FAD2A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:43:59.149721406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:43:59.152546028Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2A47FCD4C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:43:59.31299838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:43:59.315358416Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2A51B42646, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-07-02T03:43:59.59815164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:44:01.197765514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:44:01.204945974Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2AC25150CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:44:03.13021422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:44:03.134719069Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2B355C9201, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:44:03.722519481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:44:03.727198531Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2B58AAADB6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T03:44:04.597387404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:44:07.153741347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:44:07.156211468Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2C250D53C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:44:09.597602869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:44:09.642761264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:44:09.645018971Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2CB9667C6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:44:10.185266902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:44:10.187811653Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2CD9BF521D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:44:11.046627199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:44:11.049353217Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2D0D1A5AE8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:44:11.589135626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:44:11.594956385Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2D2D9324C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:44:14.597152469Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:44:14.597182198Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:44:14.598240337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:44:16.905460719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:44:16.909955733Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2E6A6B6BCE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:44:19.402429519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:44:19.407355214Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2EFF47472B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T03:44:19.597981101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:44:19.628184831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:44:19.630634423Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2F0C95A337, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:44:20.819449956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:44:20.821948443Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2F5398132B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:44:21.190982855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:44:21.193326943Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2F69BB0DFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:44:21.288533287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:44:21.291401155Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2F6F9394D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:44:22.706150462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:44:22.710639658Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2FC42B6278, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:44:23.114927973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="513 B" avg_comp="513 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:44:23.117169324Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C2FDC670026, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T03:44:24.598316888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:44:27.208424017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="536 B" total_uncomp="5.6 kB" avg_uncomp="2.8 kB" idle=1 full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:44:27.210763216Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C30D065CA29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T03:44:29.59771837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:44:30.192447677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:44:30.194850802Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C318242A710, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:44:33.43230467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:44:33.435158547Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3243668B75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:44:33.57764892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:44:33.584479653Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C324C4D2761, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-07-02T03:44:34.598290984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:44:35.962447327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:44:35.968886891Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C32DA6CAC13, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:44:36.339970123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:44:36.344322974Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C32F0CD596E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:44:36.344341566Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:44:36.344362928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:44:36.346098186Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C32F0EB6777, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:44:38.025605569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:44:38.027935889Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C335527419B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:44:38.566169836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:44:38.568398505Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C33755E65A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-02T03:44:39.597879784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:44:39.661056395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:44:39.66360653Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C33B6A307C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:44:44.578330873Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:44:44.597583951Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:44:44.597611792Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:44:44.597636166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:44:45.787115028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:44:45.789589068Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3523C78722, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:44:45.838619904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:44:45.841307682Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3526DA8307, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:44:46.478203538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:44:46.48044546Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C354CF66AE1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T03:44:49.597949243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:44:54.597961437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:44:54.672287141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:44:54.679143841Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3735A40474, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:44:54.816735353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:44:54.822809978Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C373E338CCB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:44:55.81123061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:44:55.81353846Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3779416502, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:44:56.12229989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:44:56.124653974Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C378BCD727B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:44:59.249854554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:44:59.254423341Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C38465926CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T03:44:59.597416911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:45:00.329445387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:45:00.334363219Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3886B7A5B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:45:03.42691078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:45:03.430430624Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C393F3408E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:45:03.921688025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:45:03.924024218Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C395CAE5BEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:45:04.292064114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:45:04.294429603Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3972C1BD84, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T03:45:04.597516853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:45:06.000429722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:45:06.002742514Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C39D894D911, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:45:06.787061822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:45:06.789470863Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3A077903E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:45:07.421282614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:45:07.423575822Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3A2D449947, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:45:07.670535673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:45:07.672853816Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3A3C204DA7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:45:09.063165187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:45:09.065527035Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3A8F22AC0F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:45:09.597769784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:45:11.772245819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:45:11.775051694Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3B30A38DE6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:45:13.500687902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="536 B" total_uncomp="5.6 kB" avg_uncomp="2.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:45:13.505303434Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3B97C3D5C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:45:14.597820157Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:45:14.597830699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:45:14.597853808Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:45:14.610072102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:45:14.612109416Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3BD9BEFFF3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:45:14.745518575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:45:14.748203132Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3BE1DAB4C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:45:17.393452148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:45:17.399132343Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3C7FDABB23, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:45:18.470179775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:45:18.474930913Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3CBFFB2544, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:45:19.597766385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:45:20.226599787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:45:20.228819648Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3D2885C834, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:45:22.219764115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:45:22.221986559Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3D9F52E3D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T03:45:24.59797085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:45:27.328224121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:45:27.330642069Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3ECFD214AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:45:29.255140211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:45:29.259397894Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3F42C857B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:45:29.352708036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:45:29.359111767Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3F48B3F448, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T03:45:29.597621909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:45:29.839616314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:45:29.842438076Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C3F65862360, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:45:34.501760843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:45:34.504213037Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C407B6654FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-02T03:45:34.59765504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:45:37.892937025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:45:37.895279035Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4145860A1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-07-02T03:45:39.597873327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:45:42.137396433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:45:42.145311853Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4242D8D32A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:45:42.14533394Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:45:42.145359631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:45:42.156437182Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C424382E1D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:45:43.98702343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:45:43.991710977Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C42B0E5EBDE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:45:44.48400855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:45:44.488728568Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C42CE85F9F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:45:44.488748798Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:45:44.48877207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:45:44.49582323Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C42CEF39108, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:45:44.578420463Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:45:44.597370372Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:45:44.597393214Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:45:44.597420299Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:45:46.198826618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:45:46.203763018Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4334BD8E01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-07-02T03:45:49.597988072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:45:50.652531623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:45:50.65503649Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C443E0F9307, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:45:50.898689452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:45:50.901006115Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C444CBA9805, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:45:51.034382418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:45:51.03673994Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4454CFFC85, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:45:52.802334471Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE5C44BE104135, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T03:45:54.598190068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:45:55.331637947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:45:55.335013425Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4554F77081, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:45:56.032038135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:45:56.039057241Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C457EFB0CED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:45:56.266637802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:45:56.273499591Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C458CF366A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:45:57.973696371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:45:57.981003Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C45F2B7571E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:45:58.60577119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="536 B" total_uncomp="5.6 kB" avg_uncomp="2.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:45:58.610635295Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4618407D25, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:45:58.759593353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:45:58.762429926Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C46214CAD76, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-07-02T03:45:59.598126827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:46:00.728917795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:46:00.733655969Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4696CBAB96, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:46:00.733674537Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:46:00.733695951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:46:00.735516164Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4696EAD2BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:46:00.912043143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="22 kB" avg_uncomp="11 kB" idle=1 full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:46:00.914399594Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C46A191A9D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T03:46:04.597373334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:46:04.645521912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:46:04.648293205Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C47802125BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:46:04.719451913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:46:04.724005952Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4784A41D2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:46:05.351708672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:46:05.354069032Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C47AA3246C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:46:06.724099855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:46:06.726387265Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C47FBFE9C96, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:46:09.337368642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:46:09.341786269Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4897DF6E05, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T03:46:09.597451129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:46:10.766752004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:46:10.771707569Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C48ED1BB180, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:46:11.208922856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:46:11.211452484Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4907520FB1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:46:11.543322095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:46:11.545625015Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C491B3DCB90, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:46:12.673256873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="663 B" avg_comp="663 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:46:12.675736126Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C495E98909A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:46:14.597318871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:46:14.59732695Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:46:14.597357677Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:46:16.770897717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:46:16.773324959Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4A52D570FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T03:46:19.598005029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:46:21.146350681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:46:21.148728891Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4B57A10507, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-07-02T03:46:21.148746044Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:46:21.148766091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:46:21.150523555Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4B57BE98E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T03:46:24.597906185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:46:26.227583608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:46:26.230443271Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4C868214E7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:46:26.489086522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:46:26.491532464Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4C9614C731, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T03:46:29.597657875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:46:29.883142596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:46:29.890944567Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4D60B50E8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:46:31.189905057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:46:31.194882622Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4DAE6B97DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:46:32.106358012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:46:32.110842292Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4DE504D2DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:46:32.812895175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:46:32.81797018Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4E0F2A6499, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:46:32.817989618Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:46:32.818012411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:46:32.819856488Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4E0F4AB568, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:46:32.92918954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:46:32.931464202Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4E15EF5A67, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-02T03:46:34.597345012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:46:36.823455118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:46:36.825924132Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4EFE1002AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:46:37.722284238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:46:37.7247862Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4F33A31D14, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:46:38.919520156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="536 B" total_uncomp="5.6 kB" avg_uncomp="2.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:46:38.922014798Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4F7AFED04B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T03:46:39.598132028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:46:40.031833321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:46:40.034223158Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C4FBD4A2871, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:46:41.655191908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:46:41.657912539Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C501E12A1B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:46:43.149976081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:46:43.1570075Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C50776976E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:46:44.525870654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:46:44.531472734Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C50C94E6336, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:46:44.53149313Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:46:44.531517176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:46:44.533891903Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C50C97B913C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:46:44.578373966Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:46:44.597695827Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:46:44.597731073Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:46:44.597738256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:46:44.916747281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:46:44.919056304Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C50E07435AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:46:45.501577239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:46:45.503978339Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C510350A4DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:46:46.859453922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:46:46.861937579Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5154404DF3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:46:46.920967211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="22 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:46:46.923325979Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5157EA9157, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:46:46.923340535Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:46:46.923360543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="22 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:46:46.92520606Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5158092CC4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:46:48.602527865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:46:48.605039856Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C51BC26DC55, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T03:46:49.59819901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:46:53.948718794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:46:53.951669116Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C52FAD4A80B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-07-02T03:46:54.597958022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:46:55.661570334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:46:55.668190498Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5361260B90, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:46:58.015885701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:46:58.019175651Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C53ED3B97E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:46:58.502450864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:46:58.504868665Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C540A39752D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:46:58.60837198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:46:58.610651022Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C541088D27E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:46:58.610670572Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:46:58.610690884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:46:58.612465513Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5410A7051E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T03:46:59.598059988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:47:01.03306986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:47:01.035392724Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C54A1106EE9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:47:01.265641836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:47:01.268040296Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C54AEEC6CCD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:47:01.268062013Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:47:01.268081974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:47:01.269882946Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C54AF0C9BBB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:47:03.089012389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:47:03.091323359Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C551B9AA6E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:47:03.661453995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="22 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:47:03.663755485Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C553DB86213, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T03:47:04.598042678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:47:04.619182202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:47:04.621400251Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5576CD6C54, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:47:09.320101055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:47:09.324595093Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C568F1F39DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T03:47:09.598113483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:47:09.691448289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:47:09.696275393Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C56A5498315, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:47:09.696297852Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:47:09.696318815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:47:09.698087248Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C56A5686924, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:47:12.870152452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:47:12.872514775Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C57629B7778, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:47:13.697704013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:47:13.700156007Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5793EFCD3E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:47:14.597886923Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:47:14.597917006Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:47:14.59793996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:47:14.604226812Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-07-02T03:47:14.606584827Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008457364s
level=info ts=2026-07-02T03:47:15.860459015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:47:15.862917052Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5814D955F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:47:15.862933972Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:47:15.862954409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:47:15.864765751Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5814F84337, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:47:16.443413941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:47:16.445670523Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C58379530F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:47:16.445686843Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:47:16.445706294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:47:16.447627469Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5837B479E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:47:18.083329805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:47:18.085806844Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C589956D533, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:47:18.294393308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:47:18.296717774Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C58A5EAC49A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:47:18.296734592Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:47:18.296755235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:47:18.298492915Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C58A6084D81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:47:18.417826536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:47:18.421170684Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C58AD48E42F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T03:47:19.597832206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:47:20.60880272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:47:20.616570138Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5930302061, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:47:21.651524907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:47:21.65788365Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C596E3F408A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:47:24.171860998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:47:24.175359776Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5A04415104, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:47:24.175379237Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:47:24.175400968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:47:24.1777968Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5A0471217F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-02T03:47:24.598256389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:47:26.928173663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:47:26.930632611Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5AA88850CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:47:26.930650671Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:47:26.930670642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:47:26.932571102Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5AA8A8A40F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-07-02T03:47:29.597509014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:47:29.64166376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:47:29.6461466Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5B4A648B82, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:47:30.223866387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:47:30.228869543Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5B6D200D85, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:47:30.228891489Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:47:30.228914267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:47:30.230757299Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5B6D3FF0CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:47:31.131763387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:47:31.134177568Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5BA3160321, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:47:31.74787092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:47:31.75066829Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5BC7D5AA21, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:47:31.750684874Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:47:31.750705403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:47:31.755690834Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5BC8210B2A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:47:33.356963581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:47:33.361471215Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5C27D78107, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:47:34.591444137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="527 B" total_uncomp="5.6 kB" avg_uncomp="2.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:47:34.593639971Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5C7149FFDD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T03:47:34.597759123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:47:34.818078882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:47:34.820451424Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5C7ECDA909, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:47:35.206027315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:47:35.208356819Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5C95ECF417, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:47:35.20837387Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:47:35.208391657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:47:35.210163543Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5C960BE9D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:47:35.537973359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:47:35.540780585Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5CA9B9E9F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:47:37.609655285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:47:37.611864597Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5D25316BDF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:47:39.222811617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:47:39.22574432Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5D8560CA12, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-02T03:47:39.597655208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:47:40.257393731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:47:40.262519372Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5DC32CB293, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:47:41.130962946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="22 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:47:41.133431031Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5DF7165E17, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:47:42.346636426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:47:42.349167789Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5E3F8D1313, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:47:42.349184505Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:47:42.349204506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:47:42.351110814Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5E3FADBE38, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:47:43.834736583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:47:43.837329237Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5E983FC10B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:47:44.578319236Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:47:44.59733648Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:47:44.597367456Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:47:44.597352826Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-07-02T03:47:44.597455792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:47:44.597525034Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=info ts=2026-07-02T03:47:44.597537143Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T03:47:44.597562638Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=18.525µs success=true
level=info ts=2026-07-02T03:47:44.942637601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:47:44.945068525Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5EDA485128, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:47:45.628457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:47:45.630886652Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5F03289691, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:47:45.644924293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:47:45.647048486Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5F0420D245, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:47:45.82360192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:47:45.825939551Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5F0EC87ED0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:47:46.370527338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:47:46.37285373Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5F2F62221E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:47:47.218122026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:47:47.220872744Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5F61ED93DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:47:47.222893544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:47:47.227884049Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5F6259BB24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:47:48.671220002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:47:48.676087168Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C5FB8AB56D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:47:49.597464188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:47:53.11901406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:47:53.121635708Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C60C1A23992, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:47:53.835449617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:47:53.837898624Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C60EC55916B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:47:54.105172253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:47:54.107782114Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C60FC69859D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T03:47:54.597433989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:47:57.96891395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:47:57.971362394Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C61E2B5D965, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-02T03:47:58.105664463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:47:58.107980061Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C61EAD941CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:47:59.598278671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:48:00.098603712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:48:00.102821021Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6261C0BC42, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:48:01.314892065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:48:01.32192856Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C62AA69408C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:48:02.173450142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:48:02.175895481Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C62DD514F84, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T03:48:04.597521367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:48:04.730862137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:48:04.734187101Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6375C1568E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:48:07.179447271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:48:07.182162788Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6407B47F4A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:48:08.243865433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:48:08.246239738Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6447233DB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:48:09.597813612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:48:14.597094667Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:48:14.597130911Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:48:14.598190814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:48:15.065988848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:48:15.068516072Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C65DDC6161B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:48:15.907710886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:48:15.910046112Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C660FF05E37, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:48:16.387669486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:48:17.151010567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:48:17.159530006Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C665A68D0E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:48:17.524381547Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6670280B99, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:48:18.462004848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:48:18.467082898Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C66A858D450, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-07-02T03:48:19.597428303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:48:24.38907047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="536 B" total_uncomp="5.6 kB" avg_uncomp="2.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:48:24.391556061Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C680978B04A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:48:24.391574858Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:48:24.391595743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="536 B" total_uncomp="5.6 kB" avg_uncomp="2.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:48:24.393511026Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6809996353, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T03:48:24.598124844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:48:24.897664754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="22 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:48:24.900026363Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6827C75D9F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:48:26.600218196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:48:26.603138781Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C688D4C1A1A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:48:27.504659707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:48:27.509085266Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C68C34B17E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:48:28.960936615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:48:28.963645998Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6919FB985F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T03:48:29.5979524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:48:31.706250617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:48:31.708718198Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C69BD9C1D56, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:48:31.883368561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:48:31.885804922Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C69C82A89B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:48:31.961889697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:48:31.964147101Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C69CCD75661, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:48:33.74470998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:48:33.747448546Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6A3720E80F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:48:34.412846852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:48:34.417418222Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6A5F0F6044, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T03:48:34.597259591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:48:35.0808252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:48:35.083062518Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6A86BDA05A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:48:36.732602848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:48:36.734916997Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6AE9320DC2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:48:36.746959195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:48:36.749420935Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6AEA0FC6DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-02T03:48:39.598138569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:48:42.786549102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:48:42.791912178Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6C523694F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:48:42.853965828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:48:42.856285355Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6C560EB6AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:48:42.85630262Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:48:42.856320898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:48:42.858063966Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6C562CEC7C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:48:43.806740606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="536 B" total_uncomp="5.6 kB" avg_uncomp="2.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:48:43.808982059Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6C8ED8833B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:48:43.984395889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:48:43.986699286Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6C996F8593, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:48:44.57845802Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:48:44.597652087Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:48:44.597677803Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:48:44.59769175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T03:48:49.598246165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:48:50.232096511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:48:50.234490128Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6E0DD49FE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:48:50.234512187Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:48:50.234534036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:48:50.236350245Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6E0DF45503, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:48:54.131707738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:48:54.136028761Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6EF6619499, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:48:54.136048896Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:48:54.136070561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:48:54.143246812Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6EF6D05C90, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:48:54.303720042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:48:54.308693971Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6F00AC365B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-02T03:48:54.597838689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:48:54.968038593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:48:54.97142131Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6F28208687, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:48:55.046293218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:48:55.048641855Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6F2CC8053B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:48:56.362135712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:48:56.364813434Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6F7B382006, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:48:58.386029344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:48:58.393728538Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C6FF4284D49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:48:59.178886492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:48:59.183603964Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C70233E0330, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-07-02T03:48:59.597973354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:48:59.951043294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:48:59.953688657Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C7051212EBA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:49:04.367946032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:49:04.37227047Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C7158823A5D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T03:49:04.597658361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:49:04.720998247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:49:04.725966276Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C716D9757BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:49:07.213043368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:49:07.215993927Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C720200980A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:49:07.39976394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="536 B" total_uncomp="5.6 kB" avg_uncomp="2.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:49:07.404420774Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C720D3DDFC3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T03:49:09.597343639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:49:10.231295822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:49:10.233622685Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C72B5DFE145, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:49:11.174278743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:49:11.176778163Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C72EE162C85, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:49:12.852098816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:49:12.854446719Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C73521641ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:49:14.597149233Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:49:14.597181827Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:49:14.598258535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:49:14.999196396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="22 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:49:14.999384695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:49:15.001556504Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C73D210266E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:49:15.002880699Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C73D227306A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:49:15.467724164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:49:15.472806134Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C73EE25A059, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:49:15.940076499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:49:15.942645013Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C740A26AC63, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:49:18.362105168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:49:18.364743605Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C749A85142B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=warn ts=2026-07-02T03:49:19.598211227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:49:20.937968745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="513 B" avg_comp="513 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:49:20.940291172Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C75340B19C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:49:21.837310366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:49:21.83974309Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C7569A6A0CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:49:22.982889641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:49:22.985561573Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C75ADF2B89F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:49:23.891092933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:49:23.896068145Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C75E438DCDF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:49:24.598142244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:49:27.180441803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:49:27.183146089Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C76A825F220, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:49:28.348306492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:49:28.353218126Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C76EDE29A4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T03:49:29.597919043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:49:30.672663982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:49:30.675011875Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C777845A9FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:49:32.694185039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:49:32.696530034Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C77F0C437E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:49:33.572685795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:49:33.577307002Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C782543BE4A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:49:34.251492262Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE5C784D766BA9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T03:49:34.597851266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:49:36.795445306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:49:36.800186636Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C78E55DCD0B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T03:49:39.597515077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:49:40.728756672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:49:40.731723056Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C79CFAE0AAE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:49:44.578522606Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T03:49:44.597573367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:49:44.597576977Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:49:44.597602837Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:49:44.668946736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:49:44.671638527Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C7ABA880337, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:49:47.284879782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:49:47.287265264Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C7B5671928D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:49:49.509062513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:49:49.511858319Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C7BDB07AA36, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-02T03:49:49.597832046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:49:50.303751925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:49:50.307119991Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C7C0A64BA71, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:49:51.345059142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:49:51.347722131Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C7C48748606, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:49:52.372510034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:49:52.380440325Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C7C860434FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T03:49:54.598045114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:49:56.127234357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:49:56.133443917Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C7D65B7DC9D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-07-02T03:49:59.597940264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:49:59.994753306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="536 B" total_uncomp="5.6 kB" avg_uncomp="2.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:49:59.997163154Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C7E4C02FC67, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:50:03.428421674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:50:03.433466497Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C7F18D43D74, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:50:04.276730566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:50:04.282345449Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C7F4B6CCF40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:50:04.44969526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="22 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:50:04.454351494Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C7F55ADBEB1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T03:50:04.598105636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:50:06.800801337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:50:06.803379907Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C7FE1B0AD94, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T03:50:07.361709063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:50:07.364092523Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C80031D1DE1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:50:07.726019624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:50:07.728455804Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8018D5512C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:50:07.778712092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:50:07.780904355Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C801BF6245C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:50:08.556767436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:50:08.559444128Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C804A5D5D8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-07-02T03:50:08.559462529Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:50:08.559482328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:50:08.564090767Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C804AA5FA67, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-02T03:50:09.59764875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:50:10.611323696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:50:10.615885626Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C80C4EFA0EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:50:11.976911905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:50:11.979473085Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8116361B98, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:50:11.979489981Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:50:11.979509038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:50:11.981446329Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C811656B6EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:50:12.611332723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:50:12.614270968Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C813C0C88B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:50:12.614290002Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:50:12.614310685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:50:12.619054115Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C813C56DE88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:50:14.088455069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:50:14.093221163Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C819433E1CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T03:50:14.597573222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:50:14.597568857Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:50:14.597605237Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:50:15.16177265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:50:15.164131922Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C81D409B814, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:50:15.930607273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:50:15.933013979Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8201DBC04C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:50:18.820455661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:50:18.822744969Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C82AE1B200A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T03:50:19.597588316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:50:22.914225608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:50:22.918993522Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C83A241AE80, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T03:50:24.598011204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:50:24.837303706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:50:24.843673058Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8414F916B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:50:25.143820951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:50:25.149431579Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C842732D1BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:50:27.138510271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:50:27.14362394Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C849E1035FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T03:50:27.272134245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:50:27.274332128Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C84A5DCB1C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-02T03:50:28.037923439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:50:28.040296189Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C84D383F8C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T03:50:29.597262862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:50:30.77010069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:50:30.772382121Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C85765CD3DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:50:30.877648676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:50:30.880007862Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C857CC5EEEE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:50:32.598451834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:50:32.600740407Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C85E3562277, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-02T03:50:34.597353733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:50:35.304455688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:50:35.306831702Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8684A18CFB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:50:39.598124442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:50:40.788589873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:50:40.792043626Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C87CB874A3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:50:42.098943381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:50:42.106077389Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8819E68095, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:50:44.578630152Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:50:44.597761794Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:50:44.597764016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:50:44.597790469Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:50:45.34764265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:50:45.353687494Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C88DB768404, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:50:46.708088274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:50:46.713194007Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C892C80E3D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:50:47.637740819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:50:47.642923979Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8963E8B0AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-02T03:50:49.598090188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:50:50.489118527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:50:50.491596315Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8A0DB64544, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:50:51.398095904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:50:51.400380409Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8A43E30B64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:50:51.400398223Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:50:51.400450763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:50:51.402255088Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8A4401C4E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:50:53.885721393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:50:53.888036458Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8AD829859B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:50:54.309100694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="22 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:50:54.311259176Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8AF16393E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T03:50:54.597340519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:50:54.778633635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="536 B" total_uncomp="5.6 kB" avg_uncomp="2.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:50:54.780955806Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8B0D60B497, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:50:58.925301149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="663 B" avg_comp="663 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:50:58.927765351Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8C048C75F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:50:59.342903015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:50:59.345568646Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8C1D749C8C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:50:59.427833923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:50:59.434012339Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8C22B84DD2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T03:50:59.597302113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:51:00.87824274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:51:00.880729561Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8C78F3A6DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:51:01.421634915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:51:01.424088461Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8C995807E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:51:01.472212591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:51:01.474527165Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8C9C59171D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:51:02.507197964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:51:02.512000144Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8CDA2CAAFE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T03:51:04.597256469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:51:05.203442793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:51:05.208313208Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8D7AE6BB53, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:51:07.761360712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:51:07.763728399Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8E13369B50, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:51:08.003598834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:51:08.005986088Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8E21A7B348, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=warn ts=2026-07-02T03:51:09.597829863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:51:12.834283899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:51:12.836722299Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8F4195937C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:51:13.47444763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:51:13.478605709Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8F67D97997, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:51:14.597823279Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T03:51:14.597829738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:51:14.597853661Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:51:14.729317056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:51:14.734347961Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C8FB2B1880F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-02T03:51:16.116081631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:51:16.11844403Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C9005313C9B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:51:19.597392738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:51:22.300345053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:51:22.302760269Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C9175CE891D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T03:51:22.320814245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:51:22.322903271Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C91770358C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-02T03:51:24.597468136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:51:26.671667737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:51:26.675105341Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C927A5E538B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:51:27.089213289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:51:27.091611391Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C92933F8E16, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:51:29.597612141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:51:29.811139962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T03:51:29.813583666Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C93357C342D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T03:51:34.598253322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:51:36.926678088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:51:36.93359849Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C94DDDED50D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:51:37.846463095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:51:37.85375069Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C9514B5343B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-07-02T03:51:39.598168558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:51:40.401445937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:51:40.406393816Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C95ACDDA5D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T03:51:40.75931327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:51:40.761652018Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C95C20B527D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:51:41.626213301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="536 B" total_uncomp="5.6 kB" avg_uncomp="2.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:51:41.628535542Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C95F5B74EE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:51:41.902246035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:51:41.904599281Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C96062AF48E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:51:44.113091126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:51:44.116259606Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C9689FE2DB6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-07-02T03:51:44.578508244Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:51:44.597765041Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:51:44.59779345Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:51:44.597800405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:51:45.727371048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:51:45.732544674Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C96EA52EE4F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:51:46.30315616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:51:46.307534999Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C970C99FA67, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:51:47.115041892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:51:47.120550868Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C973D1081F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:51:47.488553928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:51:47.493417299Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C9753498953, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-07-02T03:51:47.872187689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:51:47.874538747Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C976A01E3FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:51:48.727992128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:51:48.730465267Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C979D0552A6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:51:49.145719233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="22 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:51:49.148110102Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C97B5E9E02C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T03:51:49.598200502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:51:51.047049569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:51:51.049379888Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C98273D747C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T03:51:54.228218364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="559 kB" avg_uncomp="559 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T03:51:54.230692905Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C98E4DBF8F0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T03:51:54.597572862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:51:55.722894623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:51:55.725431109Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C993DF34DD9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T03:51:57.674071587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:51:57.676528755Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C99B23F0B97, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:51:58.110612885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-02T03:51:58.113154725Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C99CC43A8C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-02T03:51:59.598006054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:52:02.409594956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T03:52:02.413761318Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C9ACC9C9E01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T03:52:03.845449937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="21 kB" avg_comp="21 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-02T03:52:03.851297648Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C9B224B774D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-02T03:52:04.59808809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:52:09.006569244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-02T03:52:09.00893659Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C9C55B6C573, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-02T03:52:09.597448162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:52:09.74468987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T03:52:09.747069814Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C9C81B50B81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T03:52:14.596437928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:52:14.597047103Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:52:14.597070304Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T03:52:14.597789667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T03:52:14.598719585Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C9DA2E37EC6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T03:52:14.603203557Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-07-02T03:52:14.605493024Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007769745s
level=info ts=2026-07-02T03:52:18.761786022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="243 kB" avg_uncomp="243 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T03:52:18.764359428Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C9E9B2D5339, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T03:52:19.597517686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:52:22.788607972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="179 kB" avg_comp="179 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T03:52:22.792019623Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C9F8B33C1CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T03:52:23.881939605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T03:52:23.884339139Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C9FCC5AFE84, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T03:52:24.59753045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:52:24.627663939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="527 B" total_uncomp="5.6 kB" avg_uncomp="2.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T03:52:24.629991625Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5C9FF8CC5A32, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T03:52:28.739588233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="681 B" avg_comp="681 B" total_uncomp="1.7 kB" avg_uncomp="1.7 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T03:52:28.7419732Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5CA0EDE531E7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T03:52:29.597921723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:52:31.258209187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="373 kB" avg_comp="373 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-07-02T03:52:31.261125814Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5CA1840CC70A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-07-02T03:52:31.568774517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-02T03:52:31.574924084Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5CA196BF6B58, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-02T03:52:31.702292986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:52:31.709304858Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5CA19EC2C2B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:52:31.709324655Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:52:31.709349236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T03:52:31.718284672Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5CA19F4D5684, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:52:32.299869896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:52:32.306470575Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5CA1C25AE1D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:52:32.306490366Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:52:32.306511762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:52:32.312305244Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5CA1C2B5E986, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:52:33.259139638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T03:52:33.264044357Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5CA1FB6C5911, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T03:52:34.225218708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-07-02T03:52:34.228548046Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5CA234E7BC64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-07-02T03:52:34.597601647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:52:37.391624803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:52:37.396687163Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5CA2F1BF946D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:52:37.396706025Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T03:52:37.396724931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="531 B" avg_comp="531 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T03:52:37.398492728Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5CA2F1DFAED0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T03:52:38.040290534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="22 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T03:52:38.042474083Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5CA31840663C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T03:52:39.468937433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.7 kB" avg_uncomp="3.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T03:52:39.47135437Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5CA36D69860E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T03:52:39.597582855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:52:40.21969944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T03:52:40.222046071Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5CA39A281E5F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T03:52:40.586508892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="500 kB" avg_uncomp="500 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T03:52:40.588936263Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5CA3B00626F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T03:52:42.777041194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T03:52:42.779774644Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5CA4329C493C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T03:52:42.804817078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T03:52:42.809523837Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5CA43462706E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T03:52:44.578452706Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T03:52:44.597286096Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T03:52:44.597324031Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T03:52:44.597321671Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-07-02T03:52:44.597345635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T03:52:44.59750714Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=info ts=2026-07-02T03:52:44.597515483Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T03:52:44.597534077Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.625µs success=true
level=info ts=2026-07-02T03:52:47.311663208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="130 MB" avg_uncomp="32 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:52:47.315868741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="58 kB" avg_uncomp="58 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:52:47.318650402Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5CA541226FEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=error ts=2026-07-02T03:52:47.319937205Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5CA5413BECB1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T03:52:48.553142356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T03:52:48.55812067Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5CA58B053079, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T03:52:48.779039632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T03:52:48.783964413Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE5CA5987D9244, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"