level=info ts=2026-07-03T14:12:04.506999691Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-07-03T14:12:04.507028434Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-07-03T14:12:04.508214566Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=info ts=2026-07-03T14:12:04.510062426Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-216542e0
level=info ts=2026-07-03T14:12:04.510322246Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-07-03T14:12:04.51391073Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-07-03T14:12:04.515103399Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-07-03T14:12:04.515126077Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-07-03T14:12:04.51514096Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=4.820551ms
level=error ts=2026-07-03T14:12:04.515160767Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-07-03T14:12:04.515171401Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-07-03T14:12:04.515233543Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-07-03T14:12:04.515366331Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:12:04.515681843Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=info ts=2026-07-03T14:12:04.515852758Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-07-03T14:12:04.515887652Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
ts=2026-07-03T14:12:04.518939327Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-07-03T14:12:04.518956588Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-07-03T14:12:04.533524963Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-07-03T14:12:04.53352977Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-07-03T14:12:04.533605785Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-07-03T14:12:04.533951366Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-07-03T14:12:04.534017924Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-07-03T14:12:04.534009228Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-07-03T14:12:04.534054624Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-07-03T14:12:04.534190268Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-07-03T14:12:04.534213145Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-07-03T14:12:04.534221825Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-03T14:12:04.534262577Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=20.427µs success=true
level=info ts=2026-07-03T14:12:04.534275945Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-07-03T14:12:04.534352101Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=139.373µs errors=false
level=info ts=2026-07-03T14:12:04.534363825Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-07-03T14:12:04.534465472Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=252.853µs errors=false
level=info ts=2026-07-03T14:12:04.534472465Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-07-03T14:12:04.534477356Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=265.291µs
level=info ts=2026-07-03T14:12:04.534494948Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-07-03T14:12:04.534760379Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-07-03T14:12:04.534759901Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=14.064282686s
level=info ts=2026-07-03T14:12:04.534783982Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-07-03T14:12:04.534841939Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-07-03T14:12:04.534857396Z caller=loki.go:615 msg="Loki started" startup_time=93.033571ms
level=info ts=2026-07-03T14:12:04.534863818Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
ts=2026-07-03T14:12:05.684233906Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-07-03T14:12:05.684253042Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-07-03T14:12:08.779764194Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-07-03T14:12:08.779788346Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-07-03T14:12:09.535211582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:12:14.53527811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-07-03T14:12:14.972813713Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-07-03T14:12:14.972836442Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-07-03T14:12:19.534888743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:12:24.535899901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:12:29.535860933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:12:30.676148357Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=26.160970353s
level=warn ts=2026-07-03T14:12:34.535874577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:12:34.535874603Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:12:34.535895999Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-07-03T14:12:34.535901081Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:12:39.535158691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:12:44.535505359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:12:49.535173951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:12:54.535185275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:12:59.535759639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:13:04.515397815Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-03T14:13:04.535590209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:13:04.535610324Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:13:04.535634053Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:13:09.535360787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:13:14.535780722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:13:19.535011058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-03T14:13:21.696833157Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BECD17205F28A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-03T14:13:24.535835407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:13:29.535010805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:13:34.534902604Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T14:13:34.534908218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:13:34.534934208Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:13:39.53574264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:13:44.535773719Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:13:49.535857458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:13:54.535002311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:13:59.535273534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:14:04.515847958Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:14:04.535475778Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:14:04.535499456Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:14:04.535534597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:14:09.534914157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:14:14.535393053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:14:19.535348618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:14:24.535452012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:14:28.358056875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:14:28.370436202Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD26A66AF602, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:14:29.535616708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:14:34.535522075Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:14:34.535555547Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:14:34.535584483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:14:38.760913935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:14:38.772777039Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD291272E69E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:14:39.535845727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:14:44.535000155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:14:49.535342637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:14:54.535892726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:14:59.534886162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:15:00.025434185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:15:00.036424232Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD2E05D9EE16, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:15:04.126009377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:15:04.132683951Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD2EFA029A52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:15:04.515475897Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:15:04.534844735Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:15:04.534870987Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:15:04.534884951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:15:09.535555305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:15:14.535088485Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:15:19.534955452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:15:21.982565454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:15:21.991993249Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD3322828167, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:15:24.53488776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:15:29.535264231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:15:34.53522325Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:15:34.535258885Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:15:34.535259542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:15:39.53581823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:15:41.925284216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:15:41.936170208Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD37C7471015, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:15:44.535386307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:15:49.535394571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:15:54.535890174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:15:54.873071907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:15:54.882880269Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD3ACAF3F564, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:15:59.535535458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:16:04.5154043Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:16:04.535474181Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T14:16:04.535486277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:16:04.535501456Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:16:09.535084155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:16:14.53526976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:16:19.535336437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:16:24.535235569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:16:29.535751398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:16:29.647759076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:16:29.658950026Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD42E3C68553, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:16:34.535417473Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:16:34.535445949Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:16:34.535442882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:16:39.534888416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:16:42.449680701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:16:42.459867642Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD45DEBF441D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:16:44.535765631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-03T14:16:45.49143241Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BECD46937AF4BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-03T14:16:49.5356476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:16:54.535361471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:16:59.535364545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:17:04.515858184Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:17:04.53523499Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-07-03T14:17:04.535263959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:17:04.535258625Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:17:04.535288591Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T14:17:04.535350987Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-07-03T14:17:04.535358655Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-03T14:17:04.535371759Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=8.665µs success=true
level=warn ts=2026-07-03T14:17:09.535084721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:17:14.535683461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:17:19.534893072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:17:24.535227951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:17:25.61958699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:17:25.632857211Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD4FEC12853D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:17:29.535591298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:17:34.535302231Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:17:34.535331722Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:17:34.535330688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:17:38.76911338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:17:38.779256467Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD52FBAB1A7F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:17:39.535200054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:17:44.535389479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:17:49.535103288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:17:54.535757347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:17:59.535204969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:18:04.515701003Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:18:04.535265804Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T14:18:04.535277151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:18:04.535291316Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:18:09.535517917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:18:12.480596445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:18:12.493533038Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD5AD52B4B09, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:18:14.535300767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:18:19.535308595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:18:22.334018048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:18:22.341674694Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD5D202F737B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:18:24.535285955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:18:29.534920263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:18:34.535353659Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:18:34.535389322Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:18:34.535409498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:18:39.535132229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:18:44.535555011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:18:49.535399754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:18:54.535096025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:18:59.535375429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:18:59.888185182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:18:59.897834313Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD65DEB422C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:19:04.19133738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:19:04.197654516Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD66DF00A780, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:19:04.515427146Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:19:04.534894739Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:19:04.534918366Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:19:04.534910836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:19:09.535701145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:19:14.53524774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:19:19.53535286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:19:24.535421126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:19:29.535599836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:19:34.535062046Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:19:34.535093337Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:19:34.535084018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:19:39.535663948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:19:44.535429416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:19:45.399399257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:19:45.406781775Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD707742CA63, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:19:49.535891679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:19:53.646545923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:19:53.657100785Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD726300EB1A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:19:54.535869186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:19:59.535312613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:20:04.516122525Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:20:04.534940317Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:20:04.534978364Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:20:04.534986286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:20:09.535528178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:20:14.535395652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:20:19.535263322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-03T14:20:21.831855255Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BECD78F25F4B88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-03T14:20:24.535931766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:20:25.525818307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:20:25.532118945Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD79CEE8EB06, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:20:29.535346498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:20:34.535570905Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:20:34.535605152Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:20:34.535603574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:20:39.535135772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:20:42.286221417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:20:42.297448301Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD7DB633245A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:20:42.297473456Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:20:42.297504752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:20:42.306844741Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD7DB6C5219D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:20:44.535127713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:20:49.53548863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:20:54.464857308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:20:54.476332997Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD808C1FBB01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:20:54.535858864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:20:59.535891848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:21:04.515859359Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:21:04.535318283Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T14:21:04.535323057Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:21:04.535347074Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:21:09.534987107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:21:12.105776682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:21:12.113106878Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD84A759A2F8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:21:12.113131702Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:21:12.113157237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:21:12.118589148Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD84A7B10B12, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:21:14.535782589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:21:15.768920597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:21:15.778393824Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD8581D21F20, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:21:19.53518167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:21:22.654874619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:21:22.662545122Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD871C26E7B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:21:24.535421956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:21:29.534912473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:21:34.535454987Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T14:21:34.535467647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:21:34.535485927Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T14:21:34.689999964Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-07-03T14:21:34.692934832Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.157432508s
level=warn ts=2026-07-03T14:21:39.535399352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:21:44.53498607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:21:49.535393897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:21:54.535800693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:21:55.231176404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:21:55.237651197Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD8EB1C7DABC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:21:59.535377028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:22:04.515443894Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:22:04.535048473Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:22:04.535072473Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T14:22:04.535069143Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-07-03T14:22:04.535103464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:22:04.535180707Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-07-03T14:22:04.535188456Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-03T14:22:04.53520522Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.555µs success=true
level=info ts=2026-07-03T14:22:09.071238964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:22:09.082579659Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD91EB021B26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:22:09.535717414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:22:14.535434949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:22:19.53582347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:22:24.535111272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:22:29.535771811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:22:34.535037693Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T14:22:34.535058398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:22:34.535075973Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:22:39.535023414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:22:44.535717085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:22:49.535507827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:22:53.865622065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:22:53.872733285Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD9C58B410D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:22:54.535767893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:22:59.535893541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:23:04.515615504Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:23:04.535792132Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:23:04.53581876Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:23:04.53582623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:23:06.840544124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:23:07.496949264Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECD9F84C662AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:23:09.535642569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:23:14.534994396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:23:19.535185157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:23:24.535012874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:23:29.535078346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:23:34.535677934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:23:34.535704175Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:23:34.535731656Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T14:23:36.771519806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:23:36.778846064Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECDA6561BADCE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:23:39.535842459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:23:44.535245036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:23:49.535780401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:23:54.535447537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:23:59.535573129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:24:04.515494185Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:24:04.535449057Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:24:04.535475784Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:24:04.535499922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-03T14:24:06.574448401Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BECDAD461234B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-03T14:24:07.091575893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:24:07.101687976Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECDAD657C699A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:24:09.535541117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:24:14.535388341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:24:19.535239217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:24:24.535715564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:24:29.53564906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:24:32.137312197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:24:32.145601234Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECDB33A37097C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:24:34.535688728Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:24:34.535717024Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:24:34.535759102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:24:39.535855016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:24:44.535688011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:24:49.535874772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:24:54.53548015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:24:59.535745551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:25:02.638257325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:25:02.649152372Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECDBA545D378B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:25:04.515407849Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:25:04.535139463Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:25:04.535161265Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:25:04.535170677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:25:09.535990665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:25:14.535236021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:25:19.535692031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:25:20.543347882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:25:20.551253461Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECDBE7F6C509B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:25:24.535451592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:25:29.535141443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:25:34.535460625Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:25:34.535491702Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:25:34.535507992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:25:39.535231599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:25:44.535404979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:25:45.360993682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:25:45.372803715Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECDC446E3BD40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:25:49.535359174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:25:54.53573854Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:25:59.535863163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:26:04.515408896Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:26:04.535531241Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:26:04.535556698Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:26:04.535563852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:26:09.535685249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:26:10.66560704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:26:10.673170653Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECDCA2AE9892E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:26:14.535644098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:26:19.535878136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:26:24.535454287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:26:29.535720401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:26:34.535337567Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T14:26:34.535350573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:26:34.535373109Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T14:26:34.538619094Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-07-03T14:26:34.540273985Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.004919845s
level=warn ts=2026-07-03T14:26:39.535155144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:26:42.261755988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:26:42.27305178Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECDD18668F072, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:26:44.535396545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:26:49.535402321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:26:54.535888383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:26:59.535445381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:27:04.516249923Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:27:04.535571608Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:27:04.535595496Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T14:27:04.535594425Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-07-03T14:27:04.53562706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:27:04.535710402Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-07-03T14:27:04.535717192Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-03T14:27:04.535734131Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.22µs success=true
level=warn ts=2026-07-03T14:27:09.535016146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:27:10.529102362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:27:10.53706587Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECDD81B15628F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:27:14.535855382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:27:19.535307752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:27:24.535252959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:27:29.535826077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:27:34.535775615Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:27:34.535805248Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:27:34.535812314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:27:35.106584722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:27:35.118439072Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECDDDD43E7D34, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:27:35.118457436Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:27:35.118478331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:27:35.127149954Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECDDDD4C579A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:27:39.535929887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:27:44.535084958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:27:48.713358262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:27:48.723534351Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECDE0FF2C59A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:27:49.334348659Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BECDE12398FF06, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-03T14:27:49.535790299Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:27:54.535490169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:27:55.585537858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:27:55.591441542Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECDE2988A6AC7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:27:55.591460598Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:27:55.591482782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:27:55.596565347Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECDE298D9E6AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:27:59.534907989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:28:04.515439629Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-03T14:28:04.535002338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:28:04.535009266Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:28:04.535030417Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T14:28:06.976391591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:28:06.984511827Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECDE53F9B3105, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:28:09.535375591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:28:14.535269984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:28:18.525908317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:28:18.537876577Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECDE7F04208CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:28:19.535873597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:28:24.535025627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:28:29.535551134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:28:34.535508234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:28:34.535525429Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:28:34.535554886Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T14:28:38.841014413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:28:38.849932316Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECDECAAEF02DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:28:39.53570768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:28:44.535433379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:28:49.535880949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:28:54.535164273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:28:59.535129454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:28:59.867568041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:28:59.878127909Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECDF19051C298, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:29:04.515459274Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:29:04.53569054Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:29:04.535716647Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:29:04.535743497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:29:09.535016931Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:29:14.535892378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:29:19.535249288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:29:24.535296581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:29:27.547151043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:29:27.555350551Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECDF80202ED28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:29:29.535782075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:29:34.535424946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:29:34.535450016Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:29:34.53547937Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:29:39.535675483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:29:44.535479694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:29:49.535358319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:29:54.535284993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:29:55.490383454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:29:55.50208367Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECDFE83C41EDF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:29:59.535358104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:30:04.516103554Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:30:04.534918471Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:30:04.534945048Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:30:04.534961478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:30:09.535748583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:30:13.497373386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:30:13.5044086Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE02B4CA9140, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:30:14.535797891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:30:19.53555892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:30:24.535062959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:30:29.53540092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:30:34.535229581Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:30:34.535259092Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:30:34.535285625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:30:39.535010628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:30:43.474726097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:30:43.485542033Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE09AFCD329F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:30:44.535473073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:30:49.535919362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:30:54.535604511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:30:59.535436971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:31:04.516108953Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:31:04.535699305Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T14:31:04.535705585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:31:04.535723656Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T14:31:05.957009286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:31:05.964661751Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE0EEBAA0013, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:31:09.535336768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:31:14.535597423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:31:19.534887742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:31:24.535845265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:31:29.535109442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:31:34.442076813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:31:34.453777388Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE158DBF2292, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:31:34.53516546Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T14:31:34.535183181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:31:34.535193954Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T14:31:34.543186144Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-07-03T14:31:34.544931297Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00906607s
level=warn ts=2026-07-03T14:31:39.535799709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:31:44.535869203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:31:49.535591996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:31:54.535831171Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:31:56.749147715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:31:56.756546617Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE1ABF1A42B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:31:59.535372366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:32:04.515426126Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:32:04.535201039Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:32:04.535212793Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-03T14:32:04.535225736Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:32:04.535299845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:32:04.535344302Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-07-03T14:32:04.535354586Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-03T14:32:04.53537265Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.845µs success=true
level=warn ts=2026-07-03T14:32:09.535771344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-03T14:32:13.337434942Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BECE1E9B673315, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-03T14:32:14.535865423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:32:19.535648144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:32:23.333480903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:32:23.344176865Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE20EFD797D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:32:24.535327449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:32:29.535343162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:32:34.535249749Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:32:34.535277934Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:32:34.535268704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:32:39.535256581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:32:44.535338567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:32:47.550010636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:32:47.557650245Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE269314C433, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:32:49.534927128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:32:54.535608536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:32:59.535603685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:33:04.515453403Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:33:04.535344833Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:33:04.535367325Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:33:04.535387602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:33:09.535179482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:33:13.937932605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:33:13.94875257Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE2CB81C4336, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:33:14.53525296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:33:19.53555991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:33:24.53527383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:33:29.534925657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:33:29.877308918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:33:29.884703211Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE306DF8A1EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:33:34.535644795Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:33:34.535673611Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:33:34.535718289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:33:39.535733504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:33:44.534881119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:33:49.535172183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:33:54.535570785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:33:57.760373997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:33:57.77179458Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE36EC2BF426, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:33:57.771813473Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:33:57.771836191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:33:57.780057326Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE36ECAB2CB3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:33:59.535201633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:34:04.515415174Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:34:04.535280626Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:34:04.535305431Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:34:04.535353674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:34:09.535622298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:34:14.031295474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:34:14.041134786Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE3AB5E4CC55, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:34:14.53593284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:34:19.535544948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:34:24.535897432Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:34:26.148510354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:34:26.15836419Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE3D882249DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:34:26.158383513Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:34:26.15840357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:34:26.163601104Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE3D8875AA97, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:34:29.535060431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:34:34.535792233Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:34:34.53582351Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:34:34.535840291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:34:35.695487692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:34:35.704914065Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE3FC1291C23, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:34:39.53583688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:34:41.870200613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:34:41.876787547Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE4131088227, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:34:44.535776256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:34:49.535227712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:34:54.535230721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:34:59.535244838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:35:04.515997316Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:35:04.535383778Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:35:04.535406186Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:35:04.535427408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:35:09.535222232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:35:11.588245926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:35:11.597291046Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE481C7FC6CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:35:14.53489372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:35:19.535577944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:35:24.535805969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:35:29.535201724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:35:31.265505629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:35:31.276322388Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE4CB1787413, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:35:34.535749822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:35:34.535767086Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:35:34.535790105Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:35:39.53519388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:35:44.535244299Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:35:49.535215053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:35:54.496374832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:35:54.504567116Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE5219FA623F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:35:54.535493425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:35:59.535866881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:36:04.515484661Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:36:04.535290238Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:36:04.535315563Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:36:04.535338813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:36:09.535954424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:36:14.53539695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:36:19.53500368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:36:24.535563562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:36:26.143500644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:36:26.154543189Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE5978782DE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:36:29.535480505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-03T14:36:34.412539541Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BECE5B64B237C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-03T14:36:34.53484524Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:36:34.534872764Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:36:34.534922811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:36:34.542567681Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-07-03T14:36:34.544105511Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008574739s
level=info ts=2026-07-03T14:36:39.047248849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:36:39.055165572Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE5C79674C80, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:36:39.535268397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:36:44.535686384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:36:49.535324641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:36:54.535375102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:36:59.535810878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:37:04.515496774Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:37:04.534760405Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-03T14:37:04.5348822Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-07-03T14:37:04.534890851Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-03T14:37:04.534908532Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.488µs success=true
level=info ts=2026-07-03T14:37:04.534904177Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:37:04.53492884Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:37:04.534940647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:37:06.694127199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:37:06.705042516Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE62E976CC18, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:37:09.535409078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:37:14.535458803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:37:19.53589729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:37:24.535502856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:37:29.53537208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:37:30.150292065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:37:30.157466039Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE685F57EA2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:37:34.535393818Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:37:34.535422869Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:37:34.535420018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:37:39.535804338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:37:44.535169211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:37:49.535557606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:37:52.209995903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:37:52.221706348Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE6D82781E37, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:37:54.53519217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:37:59.535544273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:38:04.515936398Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-03T14:38:04.535494751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:38:04.535492424Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:38:04.53552106Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:38:09.535344253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:38:14.425655883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:38:14.433126711Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE72AE5D20BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:38:14.535199849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:38:19.535768683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:38:24.535619904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:38:29.535630894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:38:34.535629217Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T14:38:34.535630906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:38:34.535658907Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T14:38:36.504281657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:38:36.515507074Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE77D29716E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:38:39.535394058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:38:44.534958769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:38:49.535363366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:38:54.535697704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:38:59.535116569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:39:00.632098322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:39:00.640664736Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE7D708F66E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:39:04.516105866Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:39:04.535513439Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T14:39:04.535525799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:39:04.535537366Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:39:09.535325227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:39:14.534878734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:39:19.535001671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:39:24.535890334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:39:29.535331565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:39:30.926078161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:39:30.940174276Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE847E8D9560, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:39:34.535308523Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:39:34.535340831Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:39:34.535337191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:39:39.535415271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:39:44.534927755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:39:49.535350343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:39:54.535480434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:39:55.061991685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:39:55.070253882Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE8A1CD026FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:39:59.535891363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:40:04.515402659Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:40:04.534935591Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:40:04.534962181Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:40:04.534981156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:40:09.535291379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:40:13.949289762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:40:13.95942635Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE8E82B15D07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:40:13.959447478Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:40:13.959469664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:40:13.967768912Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE8E833447B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:40:14.53543593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:40:19.535014372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:40:24.535384993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:40:29.015884581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:40:29.027479829Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE9204D2F65B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:40:29.535838008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:40:34.535523197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:40:34.535543592Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:40:34.535565632Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=error ts=2026-07-03T14:40:38.52822574Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BECE943B1D798E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-03T14:40:39.535200769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:40:44.535468212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:40:49.535573932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:40:51.327954719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:40:51.335774232Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE9736807857, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:40:51.335795275Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:40:51.335818304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:40:51.342262236Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE9736E51348, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:40:51.627026498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:40:51.636147084Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE974867A790, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:40:54.534911486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:40:59.535804381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:41:02.709530151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:41:02.717489788Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECE99DCE7BC5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:41:04.515405471Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:41:04.534895764Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:41:04.534921099Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:41:04.53492173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:41:09.535367637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:41:14.534939268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:41:19.535207381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:41:24.534888537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:41:29.535730219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:41:34.534849439Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:41:34.534883839Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:41:34.534886687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:41:34.543554681Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-07-03T14:41:34.545084134Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010024541s
level=warn ts=2026-07-03T14:41:39.535875405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:41:41.548699376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:41:41.554884072Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEA2E7CA24ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:41:44.534901958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:41:47.416158883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.5 MB" avg_comp="3.5 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:41:47.425708192Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEA445B95348, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:41:49.535374681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:41:54.535399721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:41:59.53523053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:42:04.515673696Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:42:04.534694982Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-03T14:42:04.534813285Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-07-03T14:42:04.534821228Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-03T14:42:04.534840061Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.726µs success=true
level=info ts=2026-07-03T14:42:04.534897179Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:42:04.534922625Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:42:04.534932525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:42:09.535833141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:42:14.53548412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:42:19.535236345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:42:24.534919755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:42:29.535182397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:42:34.535246486Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T14:42:34.535255224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:42:34.535275231Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T14:42:36.365930725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:42:36.372145439Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEAFAB27E916, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:42:39.535225677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:42:43.80772061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:42:43.81734471Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEB166ED2896, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:42:44.535722778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:42:49.535828823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:42:54.535722108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:42:56.717694365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:42:56.725500873Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEB4685073D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T14:42:57.018225568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="30 kB" avg_uncomp="30 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:42:57.020797283Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEB479E9E8F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T14:42:57.318716777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:42:57.321245326Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEB48BD2F5BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T14:42:57.61807359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" idle=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:42:57.620602987Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEB49DA9CD18, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T14:42:57.918498057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:42:57.921049091Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEB4AF92B855, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-03T14:42:58.217759752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="101 B" avg_uncomp="101 B" idle=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T14:42:58.220468893Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEB4C16A5338, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-03T14:42:58.518402897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:42:58.521053053Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEB4D35645B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T14:42:58.818616004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="193 B" avg_comp="193 B" total_uncomp="274 B" avg_uncomp="274 B" idle=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:42:58.821605644Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEB4E5403CFE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T14:42:59.118505633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:42:59.121072517Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEB4F719E15A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T14:42:59.417786009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="48 kB" avg_uncomp="48 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:42:59.420507829Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEB508F328A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-03T14:42:59.535269954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:43:04.51605454Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:43:04.535840794Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:43:04.535867636Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:43:04.535917222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:43:08.598487579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T14:43:08.601254851Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEB72C28CCB3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-03T14:43:09.455872027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:43:09.458650962Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEB75F446CDD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-07-03T14:43:09.53541382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:43:09.834310071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:43:09.837170087Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEB775D3620B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T14:43:12.190334256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:43:12.193219059Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEB8024282E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T14:43:12.80609877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:43:12.808615585Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEB826F13FE2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T14:43:14.535002696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:43:14.611331312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:43:14.613956918Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEB8928C80ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T14:43:14.740002939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:43:14.742672536Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEB89A37B124, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T14:43:14.90961937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:43:14.912257699Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEB8A4530CB4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T14:43:15.254314026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:43:15.256941925Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEB8B8DEF807, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T14:43:18.300215215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:43:18.302802716Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEB96E6C174A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-03T14:43:19.53537488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:43:24.535082761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:43:26.694130538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="344 B" avg_uncomp="344 B" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:43:26.696641164Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEBB62BC5410, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T14:43:29.535915758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:43:32.232720543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:43:32.23524987Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEBCACDD7037, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T14:43:34.31712669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:43:34.319829895Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEBD291B80E7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-03T14:43:34.473187217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:43:34.477297359Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEBD327E0C24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:43:34.534885748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:43:34.53489572Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:43:34.534927068Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T14:43:35.873541459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:43:35.882537181Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEBD86411DEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:43:36.644148363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:43:36.648818374Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEBDB3EE37DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T14:43:38.660508811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T14:43:38.663224565Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEBE2BFF5780, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-03T14:43:39.535560773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:43:39.974843127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:43:39.977668475Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEBE7A587B47, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T14:43:40.069291504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:43:40.071835256Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEBE7FF53712, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T14:43:40.076869697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:43:40.079072954Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEBE8065141C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T14:43:40.511444072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:43:40.514018415Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEBE9A4FB2C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T14:43:42.676754175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:43:42.679569159Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEBF1B63C00C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T14:43:44.367074906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:43:44.369646243Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEBF8020B7DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T14:43:44.535414536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:43:49.535091047Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:43:51.53902702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:43:51.541676687Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEC12B9C7825, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-03T14:43:54.535904828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:43:59.535473366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:44:02.77424911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:44:02.77823134Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEC3C95CDD3C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T14:44:04.51542947Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:44:04.535819553Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T14:44:04.535826897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:44:04.535842514Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:44:09.534931423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:44:14.535364672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:44:17.104688149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:44:17.110064683Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEC71F9A857A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T14:44:18.104397887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:44:18.111387691Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEC75B4999EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:44:19.535626296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:44:23.787664295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:44:23.794187389Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEC8AE02B3D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T14:44:24.118208333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:44:24.120794456Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEC8C17B0F7F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-03T14:44:24.535416962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:44:26.559644926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:44:26.562473591Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEC95302557A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T14:44:27.880672375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T14:44:27.883270438Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEC9A1BDCDDF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-03T14:44:29.150693446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:44:29.155209913Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEC9ED8C25D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:44:29.535200292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:44:31.209010281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:44:31.21412086Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECECA6845B234, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-03T14:44:31.264113267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:44:31.266875004Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECECA6B6ADEF0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T14:44:34.535710336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:44:34.535707018Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:44:34.535740301Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T14:44:35.124354049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:44:35.127087127Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECECB5180DC39, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T14:44:37.895806105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="146 B" avg_comp="146 B" total_uncomp="24 kB" avg_uncomp="24 kB" idle=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:44:37.898476179Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECECBF6B24565, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T14:44:39.535240071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:44:39.577715068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:44:39.580288542Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECECC5AF04D18, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T14:44:42.649598622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:44:42.652408104Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECECD120D3AD5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T14:44:43.417601781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:44:43.420257225Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECECD3FD19080, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T14:44:44.535128656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:44:49.535334964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:44:49.858910006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:44:49.86170981Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECECEBFC17B76, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T14:44:54.535667864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-03T14:44:54.908049977Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BECECFEC8C1B6A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-03T14:44:59.535675459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:45:03.040269672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="153 B" avg_comp="153 B" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:45:03.044061134Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECED1D17BF637, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T14:45:04.515486914Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:45:04.535259092Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:45:04.535283936Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:45:04.535290366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:45:05.312888146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:45:05.3155749Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECED258E1295B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T14:45:08.61883282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:45:08.621612541Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECED31DEEF960, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T14:45:08.849292862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T14:45:08.851783878Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECED32BA7170C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-03T14:45:08.907979773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:45:08.910598173Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECED32F292FEB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-03T14:45:09.535543216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:45:12.167346189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:45:12.17454713Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECED3F1B6798D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:45:13.827545382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:45:13.832811864Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECED4548B776F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T14:45:14.535209983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:45:15.166538811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:45:15.169149518Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECED4A4327BA9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T14:45:16.900012008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:45:16.902760985Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECED50B86E53D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-03T14:45:18.319893286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:45:18.322568975Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECED560287B43, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-03T14:45:19.53574872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:45:24.535288071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:45:26.584575855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:45:26.588801906Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECED74CDC0CC7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:45:27.691590789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:45:27.697813557Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECED78EF701D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T14:45:28.707774354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:45:28.710503728Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECED7CB52045E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T14:45:28.777672624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="162 kB" avg_uncomp="162 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:45:28.780224152Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECED7CF7BC8C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T14:45:29.534921947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:45:31.498530788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:45:31.501144043Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECED871A9905D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T14:45:32.860124279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:45:32.862782147Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECED8C2D21E53, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T14:45:34.534844545Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:45:34.534876828Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:45:34.534881999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:45:35.947836967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:45:35.950472662Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECED97ADCC2D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-03T14:45:39.535211652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:45:40.548408579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:45:40.551200006Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEDA8D15ACBB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T14:45:44.535730154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:45:48.196492295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:45:48.199151497Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEDC54EFF93E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T14:45:49.53488909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:45:50.00471072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T14:45:50.007345656Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEDCC0B69BF4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-03T14:45:54.535021976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:45:56.047012079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:45:56.049837405Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEDE28DFBF4B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T14:45:59.535319209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:45:59.696624549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:45:59.699379866Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEDF0267456A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-03T14:46:02.18147931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:46:02.184281834Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEDF9683B16D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T14:46:03.333012571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:46:03.335678313Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEDFDB24FB0E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T14:46:03.754929586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:46:03.75770139Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEDFF44B91E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T14:46:04.515580459Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:46:04.535576853Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:46:04.535605687Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:46:04.535605262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:46:04.792384443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:46:04.795032444Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEE03220EE03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-03T14:46:09.535007511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:46:11.659553686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:46:11.666916804Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEE1CBB90B26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:46:14.535092056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:46:16.512914048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:46:16.519859419Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEE2ECFC40A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:46:19.535248539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:46:20.241412113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:46:20.246260068Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEE3CB17FD16, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T14:46:21.020629602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:46:21.023274322Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEE3F968DD18, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T14:46:23.176347765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:46:23.179201715Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEE479E8C196, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T14:46:23.702595863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:46:23.705241684Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEE49944E9DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T14:46:24.510581635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:46:24.513221726Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEE4C96C59E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-03T14:46:24.534914039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:46:27.950083564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:46:27.952778786Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEE5967128D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T14:46:28.215511575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:46:28.218093263Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEE5A640D545, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T14:46:29.535542613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:46:34.534912558Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T14:46:34.534918436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:46:34.534942752Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T14:46:34.541859655Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-07-03T14:46:34.543354465Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008390363s
level=warn ts=2026-07-03T14:46:39.535142636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:46:41.417105779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T14:46:41.421254606Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEE8B938CEAE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-03T14:46:44.535041393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:46:47.471420454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:46:47.474098719Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEEA21FF5BCC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-07-03T14:46:49.535863019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:46:49.99646225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:46:49.998917921Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEEAB87F1E69, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T14:46:54.535577198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:46:56.198540865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:46:56.201244351Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEEC2A2CE458, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T14:46:56.498949275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:46:56.50149103Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEEC3C13F4C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T14:46:59.102150086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:46:59.104739993Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEECD73E0D60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-03T14:46:59.535811782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:47:00.681725417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:47:00.684379325Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEED3565D9D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T14:47:04.515567231Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:47:04.534565747Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-03T14:47:04.534679059Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-07-03T14:47:04.534687003Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-03T14:47:04.53470395Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.691µs success=true
level=info ts=2026-07-03T14:47:04.534866489Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:47:04.534880398Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:47:04.534905646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:47:04.773383716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:47:04.781334808Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEEE29971B9E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:47:04.781350173Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:47:04.781370595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:47:04.791516235Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEEE2A337362, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:47:06.961151558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:47:06.965820207Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEEEABCBBED9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T14:47:07.872368028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:47:07.874863113Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEEEE1FC2963, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T14:47:09.53597921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:47:11.947742782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:47:11.95043976Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEEFD4E7F1EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T14:47:12.474754386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:47:12.477330829Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEEFF44F2E90, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-03T14:47:14.535815796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:47:15.75600434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:47:15.759940262Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEF0B7F91646, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:47:17.975933101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:47:17.986818727Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEF13CB3ACE5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:47:18.154656359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:47:18.159467792Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEF146FC758F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T14:47:18.497182538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:47:18.499709915Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEF15B463D6A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T14:47:19.419468522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:47:19.422200822Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEF19241F5C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T14:47:19.535940523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:47:22.086568886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T14:47:22.089305473Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEF2313A3575, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-03T14:47:24.535354787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:47:29.535217082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:47:33.446144741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:47:33.451315442Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEF4D670288B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-03T14:47:34.535420425Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:47:34.535448589Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:47:34.535465588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:47:38.082102545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:47:38.084750977Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEF5EAA2D43B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T14:47:39.535873573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:47:44.535613728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:47:49.535511054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:47:50.520132547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:47:50.528517153Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEF8D05677C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:47:50.678181834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:47:50.682904685Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEF8D989D55F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T14:47:52.578087487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:47:52.580881555Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEF94AABF09B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T14:47:54.123532638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:47:54.126199285Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEF9A6C6F1F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T14:47:54.534949122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:47:54.734400947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:47:54.737042702Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEF9CB2EE902, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T14:47:55.417490999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:47:55.420273753Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEF9F3E8302C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-03T14:47:59.535654545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:48:03.661466142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:48:03.664217579Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEFBDF490E1D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T14:48:04.515607895Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:48:04.53502287Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:48:04.535048422Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:48:04.535050906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:48:04.986544056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:48:04.989326289Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEFC2E436873, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T14:48:08.134435096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:48:08.137215083Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEFCE9E4AC90, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-03T14:48:09.535240037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:48:10.399664305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:48:10.402416316Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEFD70EAF357, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T14:48:10.588098633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:48:10.590890742Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEFD7C254DD7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T14:48:11.934400965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:48:11.938774752Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEFDCC7C6799, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:48:11.938797195Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:48:11.938819407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:48:11.945624253Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEFDCCE75D71, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:48:12.695591463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:48:12.700633813Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEFDF9E6DDD5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T14:48:14.535227205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:48:15.001615531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T14:48:15.004220892Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEFE8333E3DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-03T14:48:17.341853498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:48:17.344554974Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECEFF0EB31000, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-07-03T14:48:19.535931162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:48:24.535944549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:48:29.535396661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:48:29.816324407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:48:29.820660236Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF01F6554EB1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T14:48:29.885721235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:48:29.889940412Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF01FA75F37A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T14:48:34.535551364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:48:34.535558098Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:48:34.535599381Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:48:39.535949664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:48:41.349274855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:48:41.357662925Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF04A5FE13DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T14:48:43.212697813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:48:44.535776799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-03T14:48:44.656203556Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF056A993BE2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:48:46.790879843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:48:46.796014602Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF05EA233422, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:48:47.320027207Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BECF0609625A92, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-03T14:48:48.90659752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:48:48.909251835Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF06681A3D46, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T14:48:49.039517102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:48:49.042006601Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF067003E6A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T14:48:49.439050358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:48:49.441632878Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF0687D5F638, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-03T14:48:49.535395117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:48:51.32899819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:48:51.331584189Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF06F87BC0A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T14:48:52.567022269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:48:52.569716401Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF074249188A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T14:48:53.315872158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:48:53.320123959Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF076F021F93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:48:53.782991844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:48:53.788095082Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF078AE4421D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T14:48:54.535859116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:48:55.24191254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:48:55.244633236Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF07E1B76FDA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T14:48:59.535339499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:49:00.7763086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:49:00.778875194Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF092B96289B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-07-03T14:49:00.778892109Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:49:00.778911331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:49:00.781053383Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF092BB90467, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-03T14:49:02.295084562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:49:02.297648737Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF09861D5606, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T14:49:04.516149667Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:49:04.535044077Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:49:04.535067713Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:49:04.535096181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:49:06.487586262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:49:06.490217981Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF0A800250BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T14:49:07.182320873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T14:49:07.184919451Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF0AA96A7A52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T14:49:07.184937225Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:49:07.184957302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="132 B" avg_comp="132 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T14:49:07.18705151Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF0AA98D7C14, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-03T14:49:09.535739363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:49:14.224623167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:49:14.227397939Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF0C4D2D6952, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-07-03T14:49:14.535317537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:49:15.270170834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:49:15.272691502Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF0C8B7BF821, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:49:15.272707545Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:49:15.272728667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:49:15.274680221Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF0C8B9DDAFE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T14:49:19.535800924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:49:24.535439792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:49:25.91374522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:49:25.917423943Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF0F05F5FA45, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T14:49:26.390536721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="942 B" avg_comp="942 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T14:49:26.393057887Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF0F224F5467, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-03T14:49:29.535700568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:49:30.258998536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:49:30.261632473Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF1008E4DF3A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:49:30.261649439Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:49:30.261667771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:49:30.26366819Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF1009075456, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T14:49:30.287753876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:49:30.29438349Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF100ADA5B43, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:49:32.659946375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:49:32.666058235Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF10983537DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:49:32.666082295Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:49:32.666108306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:49:32.668224544Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF109859EF3C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T14:49:33.272651296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:49:33.275212486Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF10BC854854, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-07-03T14:49:33.275228189Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:49:33.275248603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:49:33.277228472Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF10BCA698BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T14:49:33.648411658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:49:33.651018946Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF10D2EB41E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T14:49:34.534932802Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:49:34.534976522Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:49:34.534953718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:49:37.837914682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:49:37.840539352Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF11CCA2DB59, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:49:37.840561175Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:49:37.840580696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:49:37.842676828Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF11CCC58F24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-03T14:49:39.535410915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:49:42.887593052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:49:42.89047041Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF12F9A1F513, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T14:49:43.076930356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:49:43.079420374Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF1304E6406D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:49:43.079437232Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:49:43.079458028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:49:43.081381279Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF130505D606, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-03T14:49:44.535433301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:49:44.822001076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:49:44.824487885Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF136CE99152, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T14:49:45.086551347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:49:45.090563355Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF137CC57971, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:49:47.618337266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:49:47.624592985Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF1413CE3733, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:49:47.624612073Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:49:47.624632357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:49:47.626640944Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF1413F10AB0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T14:49:48.948675692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:49:48.951434144Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF1462E4212A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:49:48.951453217Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:49:48.951473686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:49:48.953492069Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF146306E92B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T14:49:49.535386276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:49:49.702865217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:49:49.705767946Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF148FD934CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T14:49:49.806910489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:49:49.809442314Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF149607B381, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T14:49:50.475094972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:49:50.477762564Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF14BDDDB2C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T14:49:50.956961088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:49:50.959602645Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF14DA96A98F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-03T14:49:52.208506685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="942 B" avg_comp="942 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T14:49:52.211130668Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF152530E2D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-03T14:49:53.871547474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:49:53.874327756Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF15885148D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:49:53.874345752Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:49:53.874366678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:49:53.876376464Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF1588740AD8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-03T14:49:54.535297415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:49:56.144494274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:49:56.147166881Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF160FCB2D59, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T14:49:59.282432956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:49:59.285119187Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF16CAD41D4E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T14:49:59.53581256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:50:00.825871537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:50:00.828509053Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF1726D22037, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T14:50:02.766717562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:50:02.769486445Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF179A830971, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T14:50:04.515563526Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:50:04.535647742Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:50:04.535672069Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:50:04.535690656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:50:06.912909616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:50:06.915555533Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF1891A39E53, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T14:50:09.535768994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:50:10.331616132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:50:10.334310467Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF195D68E5E7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T14:50:10.558047319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:50:10.560642207Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF196AE7638C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-03T14:50:14.535391017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:50:16.544372518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:50:16.551394637Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF1ACFFABA6F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:50:19.534883095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:50:21.307144144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:50:21.312639461Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF1BEBC4CC6D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T14:50:24.53588919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:50:26.035312566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:50:26.039422002Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF1D05805EC9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:50:27.330991159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:50:27.335931663Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF1D52C7F52E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T14:50:27.350979697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:50:27.353592197Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF1D53D48919, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T14:50:29.535303138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:50:32.406746523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:50:32.409684467Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF1E8131AE4C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T14:50:34.178985789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:50:34.181603941Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF1EEAD11DF0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T14:50:34.53578461Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:50:34.535812566Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:50:34.535826408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:50:35.828094207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:50:35.830679992Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF1F4D1BB4CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-03T14:50:36.631014928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:50:36.634062577Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF1F7CFC4F09, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T14:50:36.767395319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:50:36.770076357Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF1F85199BCA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T14:50:39.535496676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:50:42.978260674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:50:42.980928331Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF20F74C9781, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-03T14:50:44.535409078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:50:46.667878934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:50:46.670912037Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF21D33B7ABD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T14:50:48.702886089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="942 B" avg_comp="942 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T14:50:48.70565895Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF224C84930D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-03T14:50:49.535291184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:50:52.326890138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:50:52.329712265Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF2324874855, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T14:50:54.532859287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=warn ts=2026-07-03T14:50:54.535024701Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-03T14:50:54.535610821Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF23A801C7D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T14:50:58.197936563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:50:58.200717639Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF248276DBF3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T14:50:59.535386588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:51:04.516126993Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:51:04.535284083Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:51:04.535312186Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:51:04.53533634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:51:08.704932146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:51:09.535546972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-03T14:51:10.235675271Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF274FCD778B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:51:11.893640714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:51:11.898337608Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF27B2E84097, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T14:51:12.911633479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:51:12.914340864Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF27EF76FA0F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:51:12.914358594Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:51:12.914377887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:51:12.916396207Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF27EF99AD85, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T14:51:14.535920709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:51:18.878378359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:51:18.880929098Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF29531B79BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T14:51:19.535580901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:51:19.555721715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:51:19.558330216Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF297B7B928D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T14:51:19.801600831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:51:19.804030194Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF298A20C428, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:51:19.804049675Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:51:19.804069137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:51:19.80614265Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF298A423930, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T14:51:20.842202038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:51:20.846344214Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF29C83F83D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:51:22.10618234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:51:22.111574054Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF2A13A7E3BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-07-03T14:51:24.535791601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:51:24.765319481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:51:24.767827626Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF2AB1FE2A29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T14:51:29.535150728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:51:30.878583379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:51:30.881371285Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF2C1E624D70, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T14:51:32.22118622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:51:32.223991779Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF2C6E68E325, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T14:51:33.266919633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:51:33.269680606Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF2CACBD8B62, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T14:51:34.370014145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:51:34.372658272Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF2CEE7BEC5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T14:51:34.534987166Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:51:34.535013193Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:51:34.535014964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:51:34.542349189Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-07-03T14:51:34.543804083Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008980139s
level=info ts=2026-07-03T14:51:36.593874711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:51:36.596480227Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF2D730833C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T14:51:39.535512617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:51:40.050254772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:51:40.052908336Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF2E410D68C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T14:51:41.230877994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:51:41.23356017Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF2E876C694E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T14:51:43.522561288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="924 B" avg_comp="924 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T14:51:43.525134868Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF2F10024A94, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-03T14:51:44.535648636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:51:49.535534024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:51:51.370865556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:51:51.373512719Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF30E3CFDCB9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T14:51:53.413351405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:51:53.416114266Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF315D8ED38E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T14:51:54.535120052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:51:56.893600893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:51:56.896345763Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF322CFFC8F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T14:51:59.535345739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:52:00.144582665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:52:00.147283843Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF32EEC4E5E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T14:52:04.059840287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:52:04.066812803Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF33D8650BBE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:52:04.51543817Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:52:04.53456113Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-03T14:52:04.534667533Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-07-03T14:52:04.53467559Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-03T14:52:04.534693875Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.302µs success=true
level=warn ts=2026-07-03T14:52:04.534874919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:52:04.534906892Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:52:04.534930542Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T14:52:05.834185691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:52:05.839807981Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF3442127BFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T14:52:09.535506589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:52:13.442366819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:52:13.444932562Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF36075F79A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:52:14.130467573Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BECF36303F6543, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-03T14:52:14.535664387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:52:15.359477803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:52:15.363996014Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF3679C28858, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:52:17.32514548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:52:17.329842596Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF36EEEE8E4E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-03T14:52:19.25910493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:52:19.261754875Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF376214F81B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-03T14:52:19.535846799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:52:22.677918951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:52:22.680663271Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF382DDDC5DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T14:52:24.535627183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:52:27.875067414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:52:27.877623081Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF3963A1A0C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-03T14:52:29.535930038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:52:32.229575124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:52:32.232230767Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF3A672E9ACC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T14:52:33.586659333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:52:33.589260996Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF3AB8103885, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T14:52:34.535239787Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T14:52:34.535251506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:52:34.535270137Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T14:52:37.194084161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:52:37.196845197Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF3B8F19077A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T14:52:38.408405814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:52:38.411105402Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF3BD777DC19, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-03T14:52:39.535958832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:52:39.805529113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="942 B" avg_comp="942 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T14:52:39.808329731Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF3C2ABF0BAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-03T14:52:41.833954138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:52:41.83668872Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF3CA3A794F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:52:41.836710604Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:52:41.83673093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:52:41.838801641Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF3CA3CA4F3E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T14:52:44.535482833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:52:45.143438523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:52:45.15052415Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF3D692C8089, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:52:49.535184464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:52:49.667516441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:52:49.673658386Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF3E76C5AD1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T14:52:54.535880622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:52:56.24263376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:52:56.245156301Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF3FFE77A794, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T14:52:57.908737889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:52:57.911495774Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF4061C9310B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T14:52:59.535810037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:52:59.885195584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:52:59.887922417Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF40D7960094, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T14:53:00.414457537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:53:00.417012931Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF40F7212670, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T14:53:00.616589714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:53:00.620604525Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF410343ABB0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:53:04.515606029Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:53:04.534840861Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:53:04.534865255Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:53:04.534905575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:53:05.656072536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:53:05.660234995Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF422FA5B222, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T14:53:08.579021737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:53:08.581733879Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF42DDC74004, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-07-03T14:53:09.535712155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:53:11.000529148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:53:11.003242989Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF436E1CFC04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T14:53:14.535760839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:53:17.052899251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:53:17.055725742Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF44D6DE2523, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T14:53:17.161059217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:53:17.163606433Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF44DD4D9443, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T14:53:19.534897855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:53:20.845639345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:53:20.848266857Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF45B8EC223F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T14:53:21.442165733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:53:21.444772367Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF45DC7A9060, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T14:53:23.280200138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:53:23.282739181Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF464A089231, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T14:53:24.535184381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:53:25.226422999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:53:25.233833726Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF46BE53D050, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:53:25.233851574Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:53:25.233875981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:53:25.244014135Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF46BEEF2522, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:53:27.915921026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="942 B" avg_comp="942 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T14:53:27.920381367Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF475E73F2BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-03T14:53:29.535266753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:53:32.245337326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:53:32.248120578Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF48606880C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T14:53:33.14311471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T14:53:33.148224178Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF48960CD21C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-03T14:53:34.535731685Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:53:34.535764141Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:53:34.535766684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:53:35.981066015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:53:35.990119282Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF493F715E91, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:53:39.535591996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:53:42.530934645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:53:42.536301007Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF4AC59EFFDD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T14:53:43.243130438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T14:53:43.247922938Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF4AF00BF5CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-03T14:53:44.535527768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:53:45.42324144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:53:45.427727478Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF4B71F99282, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-03T14:53:49.535735638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:53:50.395496093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:53:50.398305841Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF4C9A3E4BED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T14:53:51.748177584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:53:51.750851955Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF4CEADD430D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T14:53:52.122987719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:53:52.125717289Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF4D01350FBA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-03T14:53:54.535375269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:53:59.534937809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:54:00.006203651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:54:00.010470425Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF4ED729BABC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:54:00.135004333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:54:00.140805372Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF4EDEF162F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-03T14:54:00.995560191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:54:00.998212385Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF4F120CA504, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T14:54:03.709358037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:54:03.711881747Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF4FB3CC6F34, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T14:54:04.516329798Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:54:04.535715066Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:54:04.535738893Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:54:04.535746707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:54:09.535152211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:54:09.693497863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:54:09.696245943Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF51187D0B40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T14:54:14.335953851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:54:14.342962117Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF522D73B104, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:54:14.535435799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:54:17.913205267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:54:17.919727982Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF5302A42BE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T14:54:19.535817101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:54:19.553982862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T14:54:19.558665386Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF536455466F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-03T14:54:20.382746131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:54:20.387939845Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF5395C2C90C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T14:54:21.713890947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="942 B" avg_comp="942 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T14:54:21.716730074Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF53E4F73061, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-03T14:54:22.915200069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:54:22.917874961Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF542C8F6331, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T14:54:24.535071671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:54:29.535548263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:54:32.044247199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:54:32.046859553Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF564CB17E66, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T14:54:34.535832821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:54:34.53583784Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:54:34.535864309Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T14:54:36.042378717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:54:36.045004143Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF573AFEE918, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T14:54:39.535878234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:54:40.354748706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:54:40.357477893Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF583C09C52A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T14:54:40.909945362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:54:40.912560275Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF585D2150D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T14:54:42.425207325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:54:42.427770268Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF58B770FCE5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T14:54:43.799729734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:54:43.802339541Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF59095EAF73, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T14:54:44.535741103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:54:44.725318421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:54:44.72756098Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF594086A5E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-03T14:54:45.706088493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:54:45.710055316Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF597B144B11, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:54:45.71007569Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:54:45.71009889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:54:45.716705161Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF597B7C4657, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:54:49.194785902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:54:49.199706383Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF5A4B150473, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-03T14:54:49.535909196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:54:54.535435023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:54:59.53586326Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:54:59.804428848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:54:59.808505911Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF5CC36998B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T14:55:00.187717987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:55:00.190310018Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF5CDA2C15CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T14:55:03.048134708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:55:03.052264212Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF5D84C06341, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:55:04.515425122Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:55:04.534840744Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:55:04.53486639Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:55:04.534892387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:55:07.115803301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:55:07.121377226Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF5E774AC05F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T14:55:09.535925442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:55:09.757932598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:55:09.765040989Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF5F14DDC22C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:55:10.437578878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T14:55:10.44453387Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF5F3D5D45EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-03T14:55:14.441486891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="942 B" avg_comp="942 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T14:55:14.446545794Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF602BE1B227, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-03T14:55:14.535789284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:55:19.364372233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:55:19.36704656Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF615131B2A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T14:55:19.535197387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:55:20.261369275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:55:20.263915936Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6186A73E5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T14:55:22.482573372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:55:22.485232228Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF620B0D47D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T14:55:24.535029357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:55:24.768501121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:55:24.77114145Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF62934D2523, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T14:55:25.2286183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:55:25.23127407Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF62AEBAF871, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T14:55:26.513782751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:55:26.516442244Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF62FB54F929, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T14:55:28.782696912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:55:28.786798758Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6382A876F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:55:28.963267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:55:28.968649172Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF638D7EC6AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-07-03T14:55:28.968666327Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:55:28.968685955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:55:28.970774112Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF638DA19E69, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-07-03T14:55:29.535761842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:55:32.097018683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:55:32.099786308Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6448201FD4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T14:55:32.460879385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:55:32.463620774Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF645DCF1181, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T14:55:34.535758073Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:55:34.535786525Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:55:34.535794213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:55:36.343023369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="126 B" avg_comp="126 B" total_uncomp="10 kB" avg_uncomp="10 kB" idle=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T14:55:36.345758518Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF654534A36F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-03T14:55:39.535641535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:55:44.05963087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:55:44.062404605Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6711272746, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T14:55:44.535877861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:55:45.995909761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:55:45.998605723Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF67848F432E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T14:55:46.274422011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:55:46.276880284Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF679525B6B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-03T14:55:47.083164243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:55:47.085844016Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF67C55D7C39, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-03T14:55:49.534952904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:55:51.179321727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T14:55:51.181993982Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF68B9831084, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-03T14:55:53.326073817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:55:53.328750735Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6939780142, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T14:55:54.535317759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:55:57.429595249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T14:55:57.434642424Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6A2E32F295, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-03T14:55:58.80959066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="924 B" avg_comp="924 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T14:55:58.815912759Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6A80875574, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T14:55:58.815936675Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:55:58.815957619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T14:55:58.818037831Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6A80AA90EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-03T14:55:59.535538688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:56:01.535698181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:56:01.538494476Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6B22CE2596, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-07-03T14:56:01.538512571Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:56:01.538534688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:56:01.540707104Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6B22F2F6D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-07-03T14:56:04.390688742Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BECF6BCCD25619, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-03T14:56:04.51617346Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-03T14:56:04.535002425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:56:04.535011725Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:56:04.535040415Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T14:56:08.557899492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:56:08.565391942Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6CC5A50158, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:56:09.535860322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:56:09.962692607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:56:09.967451836Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6D193651B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:56:09.967470372Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:56:09.967490946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:56:09.969513606Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6D195901AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T14:56:09.977583398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:56:09.979708103Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6D19F40140, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:56:09.979720564Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:56:10.267579697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:56:10.270166286Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6D2B41AE49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T14:56:10.568160293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="75 kB" avg_uncomp="75 kB" idle=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T14:56:10.571452276Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6D3D35E722, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T14:56:10.868167874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="421 kB" avg_uncomp="421 kB" idle=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T14:56:10.871030602Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6D4F1231FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=info ts=2026-07-03T14:56:11.298087037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T14:56:11.300692341Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6D68AECFA8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-03T14:56:11.742654794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:56:11.745692797Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6D8331E388, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:56:11.745710766Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:56:11.74573238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:56:11.747740242Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6D83569436, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T14:56:12.113985146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:56:12.116630677Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6D99515595, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:56:12.116650072Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:56:12.116671618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:56:12.118657537Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6D9971EBDA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T14:56:13.947933625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:56:13.952027718Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6E06B681FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:56:14.182653562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:56:14.187149912Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6E14B8FE21, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:56:14.18717103Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:56:14.187199427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:56:14.18925299Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6E14DC24FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T14:56:14.535458472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:56:16.67087293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T14:56:16.673640568Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6EA8EEAB29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-03T14:56:16.765000145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:56:16.767570468Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6EAE88DE13, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T14:56:16.807758819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:56:16.810393482Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6EB116894F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T14:56:18.177377677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:56:18.179940886Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6F02B7373C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T14:56:19.535245202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:56:20.697288502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:56:20.699897248Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF6F98EB5AAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T14:56:22.991524895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:56:22.994153752Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF7021AAAFAB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T14:56:24.160497654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:56:24.1630516Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF70675684D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-07-03T14:56:24.535856509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:56:25.001769632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:56:25.004430524Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF70997D0C8A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T14:56:27.440310006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:56:27.44280892Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF712AD374DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T14:56:29.535022825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:56:30.121644627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T14:56:30.124480118Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF71CAA8CE47, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=info ts=2026-07-03T14:56:30.261792575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T14:56:30.264399103Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF71D3016DE5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T14:56:30.99854794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:56:31.001260023Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF71FEEBB219, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:56:31.00127782Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:56:31.001296293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:56:31.003285768Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF71FF0E7AA6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T14:56:31.960799592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:56:31.963612231Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF7238492CB5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T14:56:34.535575256Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:56:34.535607178Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:56:34.535599904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:56:34.543763464Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-07-03T14:56:34.545343209Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010528222s
level=info ts=2026-07-03T14:56:38.331779748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:56:38.334592993Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF73B4069D1E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:56:38.334611755Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:56:38.334631591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:56:38.336665712Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF73B428BD4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T14:56:38.788924526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T14:56:38.793736004Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF73CF6556C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-03T14:56:39.535738729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:56:41.015250063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:56:41.021723341Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF7454307404, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T14:56:42.137533594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:56:42.140266772Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF7496DC55B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:56:42.14028502Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:56:42.140306534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:56:42.14234415Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF7496FEDB28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T14:56:44.534899741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:56:44.637205905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:56:44.639821606Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF752BD8629A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T14:56:45.509553429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T14:56:45.512238367Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF755FD7B679, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-03T14:56:48.994062699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:56:48.996717732Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF762F8982C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T14:56:49.5349372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:56:50.0847564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:56:50.087432013Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF76708D2840, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T14:56:50.540004316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:56:50.542726235Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF768BB0C6BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T14:56:52.938309345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:56:53.16847342Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF7728319523, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:56:53.786178451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T14:56:53.790984262Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF774D4C8B3B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T14:56:54.031528581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:56:54.034180537Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF775BCAD6E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-03T14:56:54.535483639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:56:55.423606049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:56:55.426226871Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF77AEC45B75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T14:56:55.957687626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:56:55.96026865Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF77CE98B812, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T14:56:56.358440007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:56:56.362569196Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF77E6935B07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:56:57.512782472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:56:57.517366093Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF782B68A26D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T14:56:58.016905144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:56:58.019547233Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF78495771BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T14:56:59.414157184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T14:56:59.416864455Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF789CA00551, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=warn ts=2026-07-03T14:56:59.535198553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:57:00.117865279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:57:00.120384858Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF78C68FD7D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T14:57:04.515426691Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:57:04.535201734Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:57:04.535209591Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-03T14:57:04.535221566Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:57:04.535246791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:57:04.53531626Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-07-03T14:57:04.535323661Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-03T14:57:04.535340482Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.555µs success=true
level=info ts=2026-07-03T14:57:05.338158014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T14:57:05.340848917Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF79FDB96BCB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-03T14:57:08.449827958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:57:08.452496868Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF7AB731E8A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T14:57:09.535486541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:57:14.5349468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:57:17.507305218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:57:17.510088357Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF7CD31101CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-03T14:57:18.264906842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:57:18.267534046Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF7D0037FB7A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T14:57:19.535334628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:57:20.395570358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:57:20.398235436Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF7D7F36DFD9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-03T14:57:24.535096613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:57:26.73521198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T14:57:26.740102463Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF7EF9375918, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-03T14:57:29.535401233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:57:29.758865749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:57:29.765352671Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF7FAD881FF8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T14:57:34.535488354Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:57:34.535516548Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:57:34.535555541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:57:35.287329163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:57:36.831778124Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF8152BB05E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:57:37.628411052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T14:57:37.632903509Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF81827B6D53, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-03T14:57:37.739205813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T14:57:37.741718481Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF8188F7C3C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-03T14:57:39.535091482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:57:39.784543641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:57:39.787338366Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF8202E58345, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T14:57:42.74422086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:57:42.746911552Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF82B34D087B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T14:57:43.521125315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:57:43.523772866Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF82E19AB322, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-03T14:57:44.535283673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:57:44.780692533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:57:44.783284309Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF832CAD7979, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T14:57:46.0278164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:57:46.030273621Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF837701BD60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T14:57:46.654409952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T14:57:46.657257684Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF839C5FB909, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=info ts=2026-07-03T14:57:47.554275097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:57:47.556873143Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF83D1FEED5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:57:47.556891178Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:57:47.556910714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:57:47.558954723Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF83D2217DD8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T14:57:49.535607352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:57:50.02485464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:57:50.02878377Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF846555DAC5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:57:52.697342017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:57:52.702741003Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF8504B0F3FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T14:57:53.958962091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:57:53.961651348Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF854FC07942, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:57:53.961672868Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:57:53.961692969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:57:53.963751488Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF854FE343A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T14:57:54.535584466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:57:57.100989728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:57:57.103538881Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF860B066CFB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T14:57:59.535481915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:58:01.870270748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:58:01.873308209Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF872751B959, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T14:58:04.052846731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:58:04.055441433Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF87A9631E04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-03T14:58:04.515797604Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:58:04.535366549Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:58:04.535389803Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:58:04.535402622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:58:04.590749965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T14:58:04.593296565Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF87C972F889, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-03T14:58:07.47636838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:58:07.478843751Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF8875703BE1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T14:58:09.535280443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:58:10.696188832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:58:10.698820938Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF89355D457E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T14:58:12.743546551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:58:12.746385521Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF89AF686966, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T14:58:14.53560276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:58:19.193475547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T14:58:19.196161395Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF8B2FD87F5B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T14:58:19.329665163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:58:19.332146677Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF8B37F35C22, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-03T14:58:19.535720874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:58:19.744226374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T14:58:19.748791921Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF8B50C93E3A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-03T14:58:21.002571785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:58:21.013309334Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF8B9C26D525, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T14:58:24.535622042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:58:28.255719779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T14:58:28.260610189Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF8D4C20C4BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-03T14:58:29.535716491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:58:29.695994363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:58:29.698694104Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF8DA1D81696, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T14:58:29.878893164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:58:29.881488518Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF8DACBD47B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T14:58:30.64374809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:58:30.647766993Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF8DDA6A23F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:58:34.535113744Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:58:34.535142574Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:58:34.535163271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:58:35.062683049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:58:35.067469732Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF8EE1D8B063, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T14:58:35.607832478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:58:35.610455516Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF8F0235E315, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T14:58:37.651073581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:58:37.653640855Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF8F7BFF0157, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T14:58:38.286172058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:58:38.288765819Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF8FA1DA38A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T14:58:39.535497384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:58:42.91314034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T14:58:42.915993218Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF90B5A7038F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=warn ts=2026-07-03T14:58:44.535283824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:58:45.335702398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:58:45.33829782Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF9146086309, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-03T14:58:46.165509992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:58:46.16808692Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF91777EB32D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T14:58:48.086091921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:58:48.088828227Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF91E9F982B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T14:58:49.535795918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T14:58:54.535050611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:58:55.624230359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:58:55.626906367Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF93AB4962CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T14:58:59.535198865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:58:59.654578647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:58:59.65748052Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF949B851296, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T14:59:00.29468035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T14:59:00.297496189Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF94C1AA59CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T14:59:03.095185098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T14:59:03.097876945Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF9568974262, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-03T14:59:04.515866145Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T14:59:04.534841907Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:59:04.534863995Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:59:04.534878764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:59:09.519890831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:59:09.522527962Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF96E78709F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T14:59:09.535563675Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:59:09.937369487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T14:59:09.940180517Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF97006A55C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T14:59:11.920160419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T14:59:11.925078533Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF9776BAAE51, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-03T14:59:14.535393135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:59:16.234267017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T14:59:16.240375968Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF9877F150E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T14:59:16.541414082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:59:16.54867523Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF988A5004DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:59:17.296552153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T14:59:17.301323335Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF98B72DA3F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-03T14:59:19.535689447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:59:22.152985351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T14:59:22.155684515Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF99D8861F19, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T14:59:22.945862728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T14:59:22.948529302Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF9A07C78C0B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T14:59:23.908070357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:59:23.91072765Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF9A41222BE7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:59:23.910746817Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:59:23.910765967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:59:23.912809439Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF9A41437826, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T14:59:24.535388727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:59:24.761985206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T14:59:24.766843089Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF9A7428B5DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T14:59:25.776731054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T14:59:25.781925105Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF9AB0A6D9A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T14:59:28.595633702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T14:59:28.59818368Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF9B588747BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-03T14:59:29.535756416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:59:32.033167716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T14:59:32.036070548Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF9C256FF8D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=info ts=2026-07-03T14:59:34.535834546Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T14:59:34.535863882Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T14:59:34.535885067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:59:35.383776013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T14:59:35.386429848Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF9CED2290CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-03T14:59:36.598258933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T14:59:36.600896726Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF9D35850993, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T14:59:37.658751731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T14:59:37.661696012Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF9D74C0553E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T14:59:39.535861327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:59:41.879270486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T14:59:41.882003353Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF9E704D3AA3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T14:59:44.535443437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:59:44.843891637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:59:44.846411765Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF9F20FEE40D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T14:59:47.229943776Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BECF9FAF135FA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-03T14:59:47.873903692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T14:59:47.877149074Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECF9FD59EF82D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-03T14:59:49.535188786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:59:53.332358553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T14:59:53.335048967Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA11AF48B07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T14:59:54.535278629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T14:59:57.244136802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T14:59:57.246799444Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA2041C94D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T14:59:58.891270747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:59:58.89837731Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA2668C378C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:59:58.898397593Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T14:59:58.898422618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T14:59:58.908257425Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA26725C006, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T14:59:59.493680347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T14:59:59.498602525Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA28A545507, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T14:59:59.535638157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:00:00.44623409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:00:00.451134883Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA2C31ABD15, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:00:00.451154877Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:00:00.45117821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:00:00.457562014Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA2C37E44DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-03T15:00:03.45752071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:00:03.462339454Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA376959714, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:00:04.339322774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:00:04.341980438Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA3AB04F414, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T15:00:04.515411206Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:00:04.535281625Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T15:00:04.53529297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:00:04.53530425Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:00:07.307779188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:00:07.310579002Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA45BF61153, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T15:00:09.300773419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:00:09.304960288Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA4D2D48884, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:00:09.535426867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:00:09.934663339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:00:09.939622859Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA4F8AA576A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T15:00:11.165188816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:00:11.167785545Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA541DF1FF8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:00:11.38916409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:00:11.391692763Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA54F378A9E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:00:11.398730421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:00:11.406733101Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA5501B2011, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:00:11.881015604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T15:00:11.885821241Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA56CAA0189, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-03T15:00:12.29898457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:00:12.302025838Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA58577B35A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=info ts=2026-07-03T15:00:14.325600519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:00:14.328525737Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA5FE42B27F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-03T15:00:14.534926989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:00:16.906412264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:00:16.909154332Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA69814CCB8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:00:16.97032337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:00:16.974875795Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA69C00A8A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-03T15:00:19.535187333Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:00:21.611939101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T15:00:21.617237727Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA7B0B374B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-07-03T15:00:24.535553054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:00:27.496152988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:00:27.499090619Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA90F48BC1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:00:28.082779392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:00:28.085386406Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA9323C10B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:00:28.694117091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:00:28.69668646Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFA956AC860E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-03T15:00:29.535865096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:00:34.535499749Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:00:34.535530378Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:00:34.535570765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:00:39.535739974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:00:42.802011526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:00:42.806468013Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFAC9FADCA32, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T15:00:43.709835759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:00:43.714898942Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFACD5D18661, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-03T15:00:44.535433072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:00:44.695790241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:00:44.706258823Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFAD10EA2EC3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:00:48.989384748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:00:48.99419871Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFAE107EACFB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T15:00:49.535409644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:00:53.650406366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:00:53.653192689Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFAF26322DC8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T15:00:54.535565165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:00:54.793010152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:00:54.797026424Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFAF6A5FB16F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:00:54.79704517Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:00:54.797067728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:00:54.802933636Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFAF6ABACF4E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:00:57.974325486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:00:57.97899168Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFB028094AC8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:00:59.148521261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:00:59.151394506Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFB06DEB3164, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-03T15:00:59.535102997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:00:59.653430226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:00:59.656050939Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFB08BFEA553, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T15:00:59.753132668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:00:59.755660377Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFB091F008F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:01:03.636558744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T15:01:03.639247207Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFB1796890C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-03T15:01:04.438360155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:01:04.441015861Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFB1A933B3D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:01:04.515466631Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:01:04.535594485Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:01:04.535615095Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:01:04.535625643Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:01:04.646075194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:01:04.649404404Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFB1B59F267B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:01:05.01317783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:01:05.015847341Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFB1CB779AAF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=info ts=2026-07-03T15:01:07.210758727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:01:07.213386403Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFB24E742861, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T15:01:07.642013195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:01:07.646002393Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFB2683C0BB8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:01:09.535328492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:01:10.992658685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T15:01:10.997158361Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFB32FFA5E49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-03T15:01:12.128543998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:01:12.131521302Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFB37395D348, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-03T15:01:14.535648024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:01:18.012152629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:01:18.014700093Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFB4D24330E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T15:01:19.535401179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:01:24.53495684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:01:25.463378497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:01:25.466230206Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFB68E666E90, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:01:27.128346078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:01:27.130930918Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFB6F1A199F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-03T15:01:28.428412009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:01:28.430914184Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFB73F1DC82F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-03T15:01:29.53518926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:01:34.535627945Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:01:34.535657364Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:01:34.535655107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:01:34.541987244Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-07-03T15:01:34.543569187Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008101788s
level=info ts=2026-07-03T15:01:36.842871227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:01:36.848931023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:01:36.850371943Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFB934F33183, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:01:36.855989598Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFB9354BA4B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:01:37.070502598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:01:37.077461704Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFB9427CD630, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-03T15:01:39.535549625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:01:42.323557635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:01:42.328546483Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFBA7B7A8E0D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T15:01:42.927304343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:01:42.929918083Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFBA9F528038, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:01:44.535723007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:01:45.720501501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:01:45.723153723Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFBB45CF69D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:01:46.367073463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:01:46.369626603Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFBB6C59682C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:01:48.653634776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:01:48.656498101Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFBBF4A6A860, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=info ts=2026-07-03T15:01:48.658528404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:01:48.660616481Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFBBF4E909B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:01:49.534901043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:01:54.535245074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:01:54.618597437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:01:54.621351592Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFBD582FBB1E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T15:01:56.505228702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:01:56.507672703Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFBDC89FF862, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:01:59.13221644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:01:59.134868409Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFBE6536D83F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-03T15:01:59.535688753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:02:02.132090331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-03T15:02:02.134800026Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFBF1805B7D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-07-03T15:02:02.134820544Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:02:02.134841546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="324 B" avg_comp="324 B" total_uncomp="113 kB" avg_uncomp="113 kB" idle=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:02:02.137125038Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFBF182BB221, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:02:02.600360158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:02:02.603015463Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFBF33EE97D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:02:02.976012612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:02:02.978652854Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFBF4A5282EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T15:02:03.388737902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T15:02:03.391348106Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFBF62EB36FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-03T15:02:04.515565079Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:02:04.534766792Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-03T15:02:04.534870673Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-07-03T15:02:04.534879565Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-03T15:02:04.534879951Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:02:04.534896847Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.983µs success=true
level=info ts=2026-07-03T15:02:04.534903229Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:02:04.534906396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:02:09.484676377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:02:09.487318419Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC0CE449936, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T15:02:09.535348258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:02:14.535863397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:02:19.288798602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:02:19.291506156Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC316A3ACD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:02:19.535206133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:02:20.656457569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:02:20.659365037Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC3682A29BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:02:21.135836027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:02:21.138543619Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC384BC40B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:02:21.138559999Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:02:21.138579946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:02:21.140626244Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC384DEF1B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-03T15:02:22.238747801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:02:22.246587948Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC3C6C6E782, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:02:22.994738703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:02:23.00043376Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC3F3B4F627, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-07-03T15:02:23.000452621Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:02:23.000472742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:02:23.0026323Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC3F3DA377B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T15:02:23.219353756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:02:23.224612005Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC40112BC1A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-03T15:02:24.535316092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:02:26.857286524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:02:26.863392636Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC4D9F68CF8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:02:29.535357104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:02:32.932223129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:02:32.937135217Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC643FB6122, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T15:02:34.535592586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:02:34.535619569Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:02:34.535645635Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:02:36.960764595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:02:36.96345274Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC733F7AC91, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-03T15:02:37.382840246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:02:37.385531212Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC74D2010CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T15:02:38.788144367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:02:38.791036839Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC7A0E50042, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:02:38.791053647Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:02:38.791072017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:02:38.793172633Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC7A10A9E07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T15:02:39.51671159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:02:39.519309932Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC7CC4E9FC5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:02:39.535375438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:02:41.390208948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:02:41.393035266Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC83BFDA1DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:02:42.738324041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:02:42.74134621Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC88C5B695D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:02:42.741363469Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:02:42.741383349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:02:42.743731734Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC88C8188D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=info ts=2026-07-03T15:02:42.874961965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:02:42.877412235Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC89478E933, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T15:02:43.313877265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:02:43.316467452Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC8AEA2D1C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T15:02:44.535789521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:02:44.746507634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T15:02:44.749108809Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC90407D9DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T15:02:44.749127195Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:02:44.749148926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="924 B" avg_comp="924 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T15:02:44.751350521Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC9042BD555, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-03T15:02:47.283493739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:02:47.286078078Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC99B3EB3B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:02:47.286096733Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:02:47.286117716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:02:47.288150267Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC99B60CE74, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:02:47.611544654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:02:47.614125396Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC9AECCCF78, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:02:47.614142165Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:02:47.61416187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:02:47.616182201Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC9AEEEABA3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T15:02:48.319732845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:02:48.322280771Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFC9D9027FD7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:02:49.535179647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:02:52.697098439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:02:52.699868814Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFCADDEE12D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:02:52.699887611Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:02:52.699906986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:02:52.702028809Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFCADE116C0D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:02:53.746100146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:02:53.748687989Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFCB1C72ABA8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T15:02:53.916231452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:02:53.918946027Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFCB2698091E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=warn ts=2026-07-03T15:02:54.535297522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:02:59.535744451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:03:00.264758642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:03:00.267511767Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFCCA0FECF5E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T15:03:00.313929867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:03:00.316381208Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFCCA3E9AA0D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-07-03T15:03:00.316394908Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:03:00.316412711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:03:00.31835259Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFCCA40A58AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T15:03:02.062344864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="942 B" avg_comp="942 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T15:03:02.064960568Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFCD0C22E5AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-03T15:03:04.163517097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:03:04.166213822Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFCD89612A11, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:03:04.515416808Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:03:04.535659823Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:03:04.535684805Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:03:04.535687812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:03:04.849475895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:03:04.854230576Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFCDB26318A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-03T15:03:05.351723409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:03:05.357172723Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFCDD05D4420, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-03T15:03:05.945584361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:03:05.948233152Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFCDF398527B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:03:08.548356865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:03:08.551066846Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFCE8EBC7773, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:03:08.55108392Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:03:08.551101165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:03:08.553313049Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFCE8EE12BC3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T15:03:09.535153903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:03:13.600981197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:03:13.603729351Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFCFBBE5A384, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:03:14.535205134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:03:17.533618869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:03:17.536210906Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD0A64B9892, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T15:03:17.667350388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:03:17.669714461Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD0AE41DA55, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T15:03:18.337685453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:03:18.34013825Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD0D636AF40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-03T15:03:19.535610069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:03:22.20114052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:03:22.208346473Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD1BCC5AB40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:03:22.460987138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:03:22.467156379Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD1CC32EB5D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:03:22.517467869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:03:22.520207588Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD1CF5CD57E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=info ts=2026-07-03T15:03:24.297698439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:03:24.300528158Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD2397A4F4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:03:24.535737255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:03:26.24632711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:03:26.250502367Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD2ADB4904C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:03:27.081095286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:03:27.085446639Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD2DF796CB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:03:27.085465256Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:03:27.085485114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:03:27.087728619Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD2DF9CBC08, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:03:28.298670374Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BECFD327CBE961, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-03T15:03:28.583634382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="942 B" avg_comp="942 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T15:03:28.586217825Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD338EC6CB4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-03T15:03:29.535359379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:03:29.7246941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:03:29.727226371Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD37CEF17B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T15:03:34.000837547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:03:34.003484015Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD47BD14216, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T15:03:34.535005156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:03:34.535012585Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:03:34.535039072Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:03:35.924489988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:03:35.927178198Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD4EE7A4B7D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:03:36.110591614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:03:36.113228017Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD4F9919DA8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:03:36.113249123Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:03:36.113270978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:03:36.115455764Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD4F9B5753A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T15:03:37.051009831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:03:37.053695338Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD531A0D2FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:03:38.020450854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:03:38.023158739Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD56B688809, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:03:38.023174925Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:03:38.02319295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:03:38.02523596Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD56B8BB3AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:03:38.270864749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:03:38.273524875Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD57A54B21C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:03:39.535711945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:03:41.728953516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:03:41.731693693Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD64874E93B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:03:42.865954272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:03:42.868687007Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD68C3A4D96, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T15:03:44.535500428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:03:46.385980908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:03:46.390668466Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD75E273528, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-03T15:03:49.535066635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:03:51.688436361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:03:51.69502731Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD89A512D79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T15:03:54.535471966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:03:54.713255945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:03:54.715782265Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD94E5E9563, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T15:03:55.388911224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:03:55.393332795Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFD976A52696, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T15:03:58.823128626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:03:58.825836491Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFDA4358F690, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-03T15:03:59.029278212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:03:59.031843992Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFDA4FA128A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-03T15:03:59.534979704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:04:02.434079026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:04:02.436787296Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFDB1A9366E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T15:04:02.575326706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:04:02.578064509Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFDB22FF2EA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:04:02.923358496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:04:02.926041611Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFDB37BC5FBC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=info ts=2026-07-03T15:04:04.515749587Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:04:04.535292717Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T15:04:04.535318446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:04:04.535326043Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:04:07.205565122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:04:07.212572481Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFDC373CDA25, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:04:07.994472147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:04:07.999229425Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFDC661F968A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T15:04:09.535308312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:04:12.095681188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:04:12.098488532Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFDD5A7403E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:04:14.4179985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:04:14.422048537Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFDDE4F48C6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:04:14.535817241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:04:17.889885514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:04:17.894620651Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFDEB3F01630, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:04:19.535300245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:04:20.253234796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:04:20.255873947Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFDF40ADA9BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T15:04:24.031922028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:04:24.034553232Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFE021E82748, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-03T15:04:24.535017645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:04:25.104314963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="942 B" avg_comp="942 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T15:04:25.106895049Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFE061D32402, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-03T15:04:25.204184032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:04:25.206732173Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFE067C57B11, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:04:28.403471161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:04:28.406233964Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFE126799006, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:04:28.406254504Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:04:28.406276826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:04:28.408354296Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFE1269D6567, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:04:29.535763984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:04:29.943556221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:04:29.946315501Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFE182444181, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T15:04:31.819592707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:04:31.822382631Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFE1F2186FEE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:04:34.535411356Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:04:34.535441798Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:04:34.535448006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:04:34.785621863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:04:34.788319678Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFE2A2E1DF67, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:04:39.535234301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:04:41.311544143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:04:41.316451568Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFE427FBCCB8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-03T15:04:44.429077248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:04:44.435217109Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFE4E1E093B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:04:44.53544306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:04:48.631319299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:04:48.635330199Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFE5DC394503, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=warn ts=2026-07-03T15:04:49.535505645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:04:52.488194928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:04:52.491080407Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFE6C20AE758, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T15:04:53.370515368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:04:53.373064319Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFE6F69E638D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T15:04:53.969714041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:04:53.9723257Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFE71A5553E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-03T15:04:54.53558073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:04:58.207740601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:04:58.21180848Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFE817075F8A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:04:59.535395767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:04:59.934685821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:04:59.93934711Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFE87DFF230F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:05:01.317822258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:05:01.320388965Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFE8D0501598, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:05:01.320406579Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:05:01.320430855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:05:01.322610825Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFE8D073CCD4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:05:01.989697687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:05:01.992453584Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFE8F85EF779, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T15:05:03.047470862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:05:03.054440513Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFE937ABFB9A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:05:04.515419405Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-03T15:05:04.535737419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:05:04.535764446Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:05:04.535787674Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:05:04.66831301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:05:04.67309156Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFE998263B05, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:05:06.684768822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:05:06.687473428Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFEA1037AEB8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:05:08.87840388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:05:08.881032354Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFEA92F67194, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:05:09.182234167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:05:09.184859503Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFEAA51265C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T15:05:09.535765284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:05:11.48138195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="942 B" avg_comp="942 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T15:05:11.485799568Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFEB2E37EFA8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-03T15:05:13.023502499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:05:13.026238393Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFEB8A09ED02, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T15:05:13.107470419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:05:13.10991085Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFEB8F077AC5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T15:05:14.075507005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:05:14.078177011Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFEBC8BBAD47, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:05:14.53541456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:05:14.635733592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:05:14.638199402Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFEBEA1EDC04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:05:15.034980741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:05:15.037593795Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFEC01EBB259, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-03T15:05:19.535309631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:05:21.73626337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:05:21.739039463Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFED915B4ACE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:05:24.535483504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:05:29.535424388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:05:33.408008785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:05:33.41412995Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFF049400597, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-03T15:05:34.535567465Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:05:34.535599023Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:05:34.535613814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:05:39.535907271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:05:40.484300901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:05:40.490337187Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFF1EF06B68E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-03T15:05:41.547280889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:05:41.549988895Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFF22E2E8305, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T15:05:43.048070536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:05:43.050653724Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFF287A2B819, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-03T15:05:44.535901834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:05:45.256010382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:05:45.258837382Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFF30B400BEF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=info ts=2026-07-03T15:05:46.389816297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:05:46.392419055Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFF34ED17ED2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:05:47.089317407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:05:47.091933918Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFF378831602, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T15:05:47.691648389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:05:47.69570891Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFF39C80365E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:05:49.535714334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:05:49.902388082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:05:49.907268807Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFF42050BAF2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:05:50.728011945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:05:50.730621551Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFF45164C6C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:05:52.648291946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:05:52.651018356Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFF4C3DC442E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:05:52.836555084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:05:52.844749419Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFF4CF6854C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T15:05:54.534950756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:05:57.725131133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:05:57.730322129Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFF5F29B00D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:05:57.730342546Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:05:57.730363073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:05:57.73255373Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFF5F2C00A60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:05:58.2649376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:05:58.267509653Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFF612A16767, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-03T15:05:59.535851104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:06:00.127789118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="924 B" avg_comp="924 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T15:06:00.130465569Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFF681AB1DE7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-03T15:06:02.818589607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:06:02.821242294Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFF7220D6E5E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:06:04.51541382Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:06:04.535400857Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:06:04.535427928Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:06:04.535438435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:06:05.757160962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:06:05.759890501Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFF7D1347F26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-03T15:06:09.535783014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:06:10.487779028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:06:10.490530275Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFF8EB2CFDB4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:06:10.830159822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:06:10.832802687Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFF8FF94945C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:06:13.007234637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:06:13.009921239Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFF981589C5F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-03T15:06:14.535880627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:06:16.108444994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:06:16.113431441Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFFA3A53CA96, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:06:16.113450663Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:06:16.113474123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:06:16.121195625Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFFA3ACB606F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-03T15:06:16.269436434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:06:16.273985186Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFFA43E555EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:06:19.535842511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:06:22.722514724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:06:22.725249844Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFFBC46C1A29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-03T15:06:24.535853939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:06:27.48475634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:06:27.487393652Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFFCE045986D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-03T15:06:27.62300761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:06:27.625597055Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFFCE88182A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:06:27.660621723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:06:27.663266553Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFFCEAC09247, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=info ts=2026-07-03T15:06:28.158699072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:06:28.161312686Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFFD086FAD62, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-03T15:06:29.535389403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:06:33.556929545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:06:33.564047769Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFFE4A771933, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:06:33.564066749Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:06:33.564099194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:06:33.571167613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:06:33.572719886Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFFE4AFD6763, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:06:33.577110233Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFFE4B40C262, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-03T15:06:34.534892193Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T15:06:34.534908782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:06:34.534921239Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:06:34.545904098Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-07-03T15:06:34.54760186Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.012576112s
level=info ts=2026-07-03T15:06:38.903958402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:06:38.908943362Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFFF890B40A6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T15:06:38.917054169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:06:38.919304401Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFFF89AA7028, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:06:39.512512885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:06:39.516518794Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFFFAD438424, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:06:39.535670957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:06:39.625002429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:06:39.629528084Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BECFFFB3FFBBBE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-03T15:06:44.535830602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:06:45.370688131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:06:45.37332792Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0010A59BA7B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:06:48.029015184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:06:48.031680645Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED001A8CD44D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:06:49.092207754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="942 B" avg_comp="942 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T15:06:49.094819547Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED001E82C519C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-03T15:06:49.53530793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:06:50.195429062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:06:50.202947049Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0022A38ACE1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:06:50.348488399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:06:50.354349027Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED002333E6587, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:06:53.593349143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:06:53.595926672Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED002F475EBAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:06:54.535442628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:06:58.75815746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:06:58.76334395Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED00428757E8E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-03T15:06:59.535230142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-03T15:07:04.059101164Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BED005641EE43E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-03T15:07:04.356989093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:07:04.361673295Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED00576255955, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:07:04.515988621Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:07:04.534773179Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-03T15:07:04.534886611Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-07-03T15:07:04.534894851Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-03T15:07:04.534897005Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:07:04.534912145Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.876µs success=true
level=info ts=2026-07-03T15:07:04.534919396Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:07:04.534954136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:07:05.696605333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:07:05.699281599Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED005C5DFC758, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:07:08.601917092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:07:08.604881393Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED006730DED61, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T15:07:09.535411891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:07:09.760926878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:07:09.7635112Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED006B8200373, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T15:07:10.32015431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:07:10.322699945Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED006D9747283, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:07:11.386729948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:07:11.389286133Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0071907459A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-03T15:07:14.535355796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:07:14.790892979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:07:14.799131244Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED007E442371F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:07:15.695078543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:07:15.701548114Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0081A0E8EFB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-03T15:07:17.564869319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:07:17.567684235Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0088949B9AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=warn ts=2026-07-03T15:07:19.535251217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:07:19.547390953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:07:19.549913023Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED008FF700C63, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T15:07:20.271156807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:07:20.275153794Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0092AAA1B72, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:07:20.275173756Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:07:20.275194512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:07:20.281294296Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0092B0698BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:07:20.388654802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:07:20.393456879Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED00931B5A297, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T15:07:22.125157469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:07:22.127716678Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0099916C622, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-03T15:07:24.535912758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:07:25.731158246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:07:25.733820813Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED00A70074FC2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:07:29.535616725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:07:32.443238462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:07:32.446246283Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED00C001D5027, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:07:34.535531678Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:07:34.535557827Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:07:34.535557031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:07:37.179904648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:07:37.18429403Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED00D1A875123, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:07:39.535685797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:07:39.740516537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:07:39.746738441Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED00DB3430F60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:07:43.761303933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:07:43.764051536Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED00EA2B5D6C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:07:44.535637475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:07:44.568713684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="942 B" avg_comp="942 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T15:07:44.571209415Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED00ED2D2F332, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-03T15:07:47.03400715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:07:47.03658389Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED00F65C63B03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:07:48.361127206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:07:48.363834277Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED00FB4E10B68, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T15:07:49.535280042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:07:54.535248298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:07:54.591387158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:07:54.596440532Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED011285D4C2A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-03T15:07:57.571922106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:07:57.578233035Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED011DA18C59A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=info ts=2026-07-03T15:07:58.66429483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:07:58.666849629Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0121AFDC060, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-03T15:07:59.535872543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:08:00.151431397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:08:00.15412746Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED01273A29DAF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:08:00.820279139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:08:00.82724766Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0129BC22848, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:08:04.515416364Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:08:04.534863585Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:08:04.534884898Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:08:04.534897472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:08:06.001629515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:08:06.006216993Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED013D072575E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:08:06.058418673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:08:06.060843777Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED013D3B5E3C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T15:08:08.017065862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:08:08.019617425Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED01448756066, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:08:09.075958016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:08:09.078711619Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0148795FA6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-03T15:08:09.535759378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:08:09.830254069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:08:09.834390575Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED014B4A08578, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:08:13.818620129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:08:13.823130454Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED015A25F6C4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:08:14.183006751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:08:14.18602978Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED015B7FE9467, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:08:14.418847434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:08:14.421499076Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED015C609D872, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-03T15:08:14.535494027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:08:15.587068345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:08:15.589719237Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0160BABB377, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T15:08:17.853541344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:08:17.856191159Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED01692C2FE09, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-03T15:08:19.535511515Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:08:24.535290395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:08:25.634466135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="942 B" avg_comp="942 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T15:08:25.637209549Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED018628BDCC2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-03T15:08:26.241821373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:08:26.244478511Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED01886BD7252, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T15:08:29.535381041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:08:34.535698519Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:08:34.535730654Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:08:34.53575099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:08:38.090459451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:08:38.094120527Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED01B490874B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:08:39.535231233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:08:41.727930438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:08:41.735737279Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED01C2218404D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:08:42.657268244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:08:42.663790141Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED01C596979E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:08:42.685817705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:08:42.690573582Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED01C5B02672F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-03T15:08:44.508831762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:08:44.513456079Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED01CC7A86D5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:08:44.513474951Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:08:44.513495448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:08:44.515503318Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED01CC7CB00C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-03T15:08:44.535782485Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:08:44.944933942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:08:44.947598097Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED01CE1898ED1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:08:48.966007745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:08:48.968803149Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED01DD137E85D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:08:49.535512968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:08:50.325922948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:08:50.328663327Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED01E2245AC29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:08:50.328682797Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:08:50.328701432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:08:50.330931518Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED01E226A8D81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=warn ts=2026-07-03T15:08:54.535329608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:08:54.561426516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:08:54.564001681Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED01F1EB7EDC3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T15:08:57.515981176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:08:57.518529751Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED01FCED348F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:08:57.518545246Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:08:57.518563729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:08:57.520504688Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED01FCEF3B907, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-03T15:08:57.68198718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:08:57.684431621Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED01FD8B700FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:08:57.684446747Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:08:57.818873426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:08:57.821343821Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED01FE0DFE3E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T15:08:58.799932369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:08:58.80245762Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0201B5AC2D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-03T15:08:59.535450674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:08:59.940576358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:08:59.943142038Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0205F573B67, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:09:00.465164722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:09:00.469427235Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0207EB5ECE3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:09:02.404847327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:09:02.411948869Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED020F27DC54D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=info ts=2026-07-03T15:09:04.132515399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:09:04.13512996Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED02159336D30, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:09:04.515498404Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:09:04.535363027Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:09:04.535388187Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:09:04.535407035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:09:07.371685465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="942 B" avg_comp="942 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T15:09:07.374368983Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0221A45E139, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-03T15:09:07.37438863Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:09:07.374408758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T15:09:07.376452757Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0221A68E86D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-03T15:09:07.682411708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:09:07.684850692Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0222CC8FCC8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-07-03T15:09:07.684867213Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:09:07.684885976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:09:07.68701456Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0222CEB1E68, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T15:09:09.451946377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:09:09.454487704Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED022964364F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:09:09.535224308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:09:09.917547262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:09:09.920852928Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED022B20F6F35, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:09:11.076503674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:09:11.079203911Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED022F719C31E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T15:09:11.46400605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:09:11.46655408Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0230E30F84A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:09:11.466573412Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:09:11.46659025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:09:11.468571526Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0230E5246F8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T15:09:14.535541396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:09:15.060489606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:09:15.063331354Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED023E49222EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T15:09:16.817944374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:09:16.820708416Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0244D524E89, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-03T15:09:19.535393046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:09:19.606682537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T15:09:19.609336223Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED024F38974D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-03T15:09:21.640704652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:09:21.643676739Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0256CCA624B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-03T15:09:24.535790996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:09:24.978185374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:09:24.980815953Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED02633B39523, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:09:24.980832385Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:09:24.980852029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:09:24.983004427Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED02633D6F581, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:09:28.500559815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:09:28.50352944Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED02705AB17AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T15:09:29.535345734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:09:31.617200172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:09:31.619956427Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED027BF6CD98E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:09:32.408235098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:09:32.413154128Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED027EEB3013F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-03T15:09:34.272457944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:09:34.278565005Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0285DE43D3D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-03T15:09:34.535822506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:09:34.535836622Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:09:34.535861117Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:09:37.301751445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:09:37.30442194Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED029123EF55F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:09:37.716142376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:09:37.718805133Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0292AF17599, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:09:38.038550318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:09:38.041549543Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0293E2E9170, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=info ts=2026-07-03T15:09:39.487516718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:09:39.490320937Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0299486D6AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:09:39.53537183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:09:39.720939247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:09:39.72788133Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED029A2B18B5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:09:43.260273017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:09:43.265403657Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED02A758AB9C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-03T15:09:44.535084289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:09:49.535437843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:09:50.114300934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T15:09:50.116781266Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED02C0DECAD95, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-03T15:09:50.883880137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:09:50.886491104Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED02C3BCDA937, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:09:50.886508524Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:09:50.886528422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:09:50.888536209Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED02C3BEF2E9F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:09:51.658252147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:09:51.661267359Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED02C69FAA270, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-07-03T15:09:51.66128582Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:09:51.661307733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:09:51.663495153Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED02C6A2025B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T15:09:52.179299603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:09:52.182031324Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED02C89059815, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:09:52.182052011Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:09:52.182071676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:09:52.184620888Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED02C8927D6EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-03T15:09:54.53542132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:09:55.51289509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:09:55.515496585Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED02D4FB6758A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-03T15:09:55.519609547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:09:55.52315934Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED02D502CDAAB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:09:56.608812711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:09:56.615391831Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED02D9144ADA7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-03T15:09:59.535013824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:10:04.209384748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:10:04.211866492Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED02F560EBD34, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:10:04.516215327Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:10:04.535833518Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:10:04.535860099Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:10:04.535881497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:10:05.095166983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:10:05.097996067Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED02F8ADED30D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:10:05.407136384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:10:05.409694672Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED02F9D749E8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T15:10:05.810753233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:10:05.813406682Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED02FB58378C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:10:07.455457512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:10:07.457951761Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED030178A2107, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:10:07.457983278Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:10:07.458001199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:10:07.459941767Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED03017AB949D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T15:10:09.535354507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:10:10.83586407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:10:10.838511633Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED030E10918E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:10:10.838531397Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:10:10.838549869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:10:10.840608574Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED030E12BB33A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T15:10:12.143760285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:10:12.146680436Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0312F01A382, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-03T15:10:14.535898983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:10:17.016157511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:10:17.018826679Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED032516960A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:10:19.535198953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:10:19.662517258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:10:19.665362915Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED032EF265962, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T15:10:24.535380105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:10:26.192330676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.6 MB" total_uncomp="65 MB" avg_uncomp="33 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:10:26.19941045Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED034749D18A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:10:27.604133066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:10:27.610249565Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED034C8B5383B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-03T15:10:29.535314023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:10:29.698597802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:10:29.701158125Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0354556EA35, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T15:10:30.07721211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:10:30.079705563Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0355BE74E80, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:10:30.079722887Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:10:30.079741263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:10:30.081705186Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0355C07B9A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:10:30.702223258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:10:30.704749731Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0358127567F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:10:31.284118599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:10:31.288772345Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED035A3F71694, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-03T15:10:32.119422483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:10:32.12425513Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED035D5C21DF8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=info ts=2026-07-03T15:10:34.401995545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:10:34.404583753Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0365DAF0540, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T15:10:34.535117821Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:10:34.535143215Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:10:34.535155358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:10:37.036218042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:10:37.039165647Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED036FAB68C3B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:10:37.266010062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:10:37.268679415Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0370864C65A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:10:39.40103268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:10:39.403762601Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED03787A796B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T15:10:39.535899088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:10:41.977431377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T15:10:41.980292407Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED038213ABBEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-03T15:10:42.389418842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:10:42.3936527Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED03839DE008F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:10:44.535111323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:10:46.026162395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:10:46.032383972Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED03912C086F0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T15:10:49.257614749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:10:49.26021444Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED039D3255696, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-03T15:10:49.535488377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-03T15:10:49.612728327Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BED039E82C1CD4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-03T15:10:49.666802501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:10:49.669275202Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED039EB87A296, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:10:49.786360542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:10:49.788768462Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED039F2A76FCE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T15:10:53.556150789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:10:53.558862885Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED03AD35E2A29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T15:10:54.535927808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:10:54.558063932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:10:54.560685645Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED03B0F13CC14, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:10:57.934900609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:10:57.937609113Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED03BD85B996E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:10:59.535872864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:11:00.124511423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:11:00.127233124Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED03C5ADE94C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T15:11:04.515413508Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:11:04.535399569Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:11:04.53542224Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:11:04.535460022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:11:04.604879441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:11:04.607534469Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED03D65EAEFA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:11:09.05442153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:11:09.057321696Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED03E6F23BB21, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-03T15:11:09.53590229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:11:12.227113592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:11:12.230052834Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED03F2C400A72, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-03T15:11:14.535323986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:11:17.903858071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:11:17.906743413Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0407E9AD62F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=warn ts=2026-07-03T15:11:19.535328834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:11:21.463362546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:11:21.471521859Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED04153159BF3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:11:21.964577759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:11:21.972039803Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED04170EB110A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-03T15:11:23.311664742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:11:23.31615948Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED041C1099F0E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T15:11:24.535413461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:11:26.582008908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:11:26.584595987Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED04283DA8436, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:11:28.734488092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:11:28.737208462Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED04304281323, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T15:11:28.831400787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:11:28.835726703Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0430A07AB88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:11:29.535699268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:11:29.537849969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:11:29.542310864Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0433424D944, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:11:29.542327107Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:11:29.542346435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:11:29.544646966Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED04334496323, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:11:29.657883599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:11:29.660350353Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0433B2EC873, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T15:11:30.513499197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:11:30.516006461Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0436E2EB136, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-03T15:11:32.631992306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:11:32.634595415Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED043EC75855A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:11:32.634613591Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:11:32.634634104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:11:32.636669213Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED043EC98329E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:11:34.535586897Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:11:34.535616329Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:11:34.535626397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:11:34.542448024Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-07-03T15:11:34.544193514Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009138014s
level=info ts=2026-07-03T15:11:34.641117046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:11:34.643711052Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0446435BB7C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T15:11:37.211766042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:11:37.214319842Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED044FD6EED3D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:11:37.989228762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:11:37.991853788Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0452BC6F4DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:11:39.124694955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T15:11:39.127361394Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0456F75313C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-07-03T15:11:39.535339204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:11:43.869009695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:11:43.871756559Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0468A3D252A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:11:44.535420261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:11:47.144479841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:11:47.147242563Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0474D7A6484, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:11:49.535738809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:11:52.442015637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:11:52.444768924Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED048893CED8A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:11:54.535609007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:11:57.994648163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:11:57.997462002Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED049D43265A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:11:59.360083899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:11:59.36270751Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED04A25940CC0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-03T15:11:59.535862266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:12:04.292208159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:12:04.295074515Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED04B4B908267, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T15:12:04.443741463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:12:04.446507018Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED04B549797D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:12:04.515494397Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:12:04.535557909Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-03T15:12:04.535572145Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T15:12:04.535580835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:12:04.53559917Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:12:04.535657652Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-07-03T15:12:04.535664554Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-03T15:12:04.535680108Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.471µs success=true
level=info ts=2026-07-03T15:12:08.252354271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:12:08.257619802Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED04C37BF47B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-03T15:12:09.53577214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:12:12.65582625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:12:12.66154323Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED04D3E3FBC08, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=info ts=2026-07-03T15:12:13.466612393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:12:13.473919601Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED04D6EAD5947, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T15:12:14.535093362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:12:14.764665668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:12:14.769028169Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED04DBBDE134E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:12:15.327615778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:12:15.330231915Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED04DDD5106FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:12:17.319951376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:12:17.322701522Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED04E5413B67C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:12:17.520601988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:12:17.523158663Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED04E60071C2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T15:12:18.527363228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:12:18.529939561Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED04E9C08F2B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-03T15:12:19.53567067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:12:21.401181434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:12:21.40578458Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED04F4772FB1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:12:21.468592766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T15:12:21.473393316Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED04F4B7A96F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-03T15:12:23.276186221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:12:23.278832811Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED04FB71734DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:12:23.391172646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:12:23.393643815Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED04FBDEFCEF8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T15:12:23.671549799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:12:23.674058535Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED04FCEA6F583, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:12:24.534892061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:12:25.293881204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="260 kB" avg_comp="260 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" max_age=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:12:25.301010161Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0502F9EDB77, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:12:25.408467302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:12:25.413316609Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0503650F988, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T15:12:25.594203168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" max_age=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:12:25.598000436Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0504151A6B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:12:25.894676195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="486 kB" avg_uncomp="486 kB" max_age=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-03T15:12:25.900702517Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED050535E0A27, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-03T15:12:29.53590152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:12:34.535713298Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:12:34.53574323Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:12:34.53574725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:12:34.764593689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:12:34.767127423Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED05263D81849, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:12:37.262716033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:12:37.26628291Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED052F8CECAAB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-03T15:12:39.535868952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:12:40.665288365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="486 kB" avg_uncomp="486 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-03T15:12:40.671390517Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED053C3C3CB7F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-03T15:12:41.797427926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:12:41.80050071Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED054070B364E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:12:42.086269583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:12:42.089905421Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED05418501497, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-07-03T15:12:44.534898264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:12:47.353789414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:12:47.358766276Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED055525D0F19, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:12:47.358786429Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:12:47.358807681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:12:47.361182928Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0555284F679, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:12:47.437534989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:12:47.439931278Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED05557343FBA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-03T15:12:49.535697868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:12:54.535276933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:12:54.58537549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:12:54.590378677Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED05701672C46, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-07-03T15:12:54.590396743Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:12:54.590419533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="237 B" avg_comp="237 B" total_uncomp="130 kB" avg_uncomp="130 kB" max_age=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:12:54.594549227Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED05701A913CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:12:54.891305933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="73 kB" avg_comp="73 kB" total_uncomp="915 kB" avg_uncomp="915 kB" max_age=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:12:54.894449279Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0571386247C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:12:55.177880074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:12:55.180482349Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED05724940853, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T15:12:57.013853969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:12:57.016708489Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED05792060B35, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=info ts=2026-07-03T15:12:57.234381245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:12:57.236857538Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0579F25C8C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:12:57.742355447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:12:57.74951882Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED057BDB4F12F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:12:57.749537273Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:12:57.749560986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:12:58.164843124Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED057D67582CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T15:12:59.53580338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:12:59.790447577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:12:59.794832759Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED058379D971D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:12:59.840382339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:12:59.842749374Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0583A7974BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:13:00.028149533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:13:00.030734796Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED05845AC28FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:13:02.189665553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:13:02.19260078Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED058C685EAEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-03T15:13:04.485918929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:13:04.488642339Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0594F62A0EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:13:04.515431461Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:13:04.535000678Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:13:04.53502148Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:13:04.535042464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:13:06.552154181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:13:06.55467775Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED059CA88B019, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T15:13:07.285490859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:13:07.288059752Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED059F63E8821, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:13:07.523730874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:13:07.526499028Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED05A04768251, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:13:08.043635563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:13:08.048014745Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED05A238A5489, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:13:09.024167185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:13:09.031243244Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED05A5E260024, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T15:13:09.535199005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:13:09.958691559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:13:09.963108775Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED05A95B0C768, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:13:10.04385585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T15:13:10.046252822Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED05A9AA63A82, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-03T15:13:11.598414506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="486 kB" avg_uncomp="486 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-03T15:13:11.601253644Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED05AF7544A99, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-03T15:13:12.947008898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:13:12.94974605Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED05B47B489CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=warn ts=2026-07-03T15:13:14.535600676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:13:16.894641613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:13:16.897952707Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED05C330A376B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:13:17.73766292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:13:17.742343876Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED05C655EBC52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T15:13:19.318477669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:13:19.322308194Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED05CC38B6C30, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:13:19.535053071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:13:22.466697494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:13:22.471513103Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED05D7F3F7C84, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-03T15:13:24.535092612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:13:27.458409702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:13:27.460929882Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED05EA8A53A10, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:13:28.142898819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:13:28.14557612Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED05ED1736E96, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:13:29.535418986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:13:34.535525143Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:13:34.535555136Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:13:34.535550342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:13:34.672033816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:13:34.674656474Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED060569E58C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:13:35.875446324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:13:35.882295044Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0609E98DEDD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:13:37.925007853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:13:37.929536378Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED061189E4289, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=warn ts=2026-07-03T15:13:39.535592155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:13:40.784171144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:13:40.786884159Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED061C2EDEB35, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T15:13:43.148076462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:13:43.150850304Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0624FD5A0B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-03T15:13:44.535605582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:13:46.146296371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:13:46.149126521Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0630289C695, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:13:47.24426292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:13:47.246954348Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06343FB2427, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:13:47.255980809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:13:47.258381827Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06344AB8D65, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=warn ts=2026-07-03T15:13:49.535529276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:13:52.064016112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:13:52.066751856Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06463442177, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:13:52.516516355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:13:52.519156049Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0647E3A9D80, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:13:53.121717294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:13:53.124314256Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED064A24D3142, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T15:13:53.886194727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:13:53.88875971Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED064CFDE5FFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:13:54.535373999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:13:56.10474669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:13:56.107265547Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0655419967E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-03T15:13:59.535406843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:14:00.06592616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:14:00.068633524Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED066403752D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-03T15:14:00.336834099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:14:00.340174025Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06650666B07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:14:00.897272098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:14:00.903256768Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06671F472BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:14:01.08864428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T15:14:01.091188457Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0667D2941BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-03T15:14:03.064707002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a2f3450d55b25dc2 immediate=false num_chunks=1 total_comp="162 B" avg_comp="162 B" total_uncomp="72 kB" avg_uncomp="72 kB" max_age=1 labels="{stream=\"minio-operator\"}"
level=error ts=2026-07-03T15:14:03.067429017Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED066F2F539DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"minio-operator\"}"
level=info ts=2026-07-03T15:14:03.445604834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:14:03.449777728Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06709BDF0FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:14:03.44979795Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:14:03.449820199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:14:03.455321348Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0670A14ACC3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:14:04.016906575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:14:04.021632977Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0672BD53BF3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:14:04.515439737Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:14:04.534850945Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:14:04.534877445Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:14:04.534884677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:14:06.191726728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:14:06.196137136Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED067AD71031A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T15:14:08.330633775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:14:08.333231294Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0682CD304E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-03T15:14:09.535720829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:14:10.161410222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="486 kB" avg_uncomp="486 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-03T15:14:10.164151737Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06899F31E3D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-03T15:14:11.010356208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:14:11.013015188Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED068CC8D74C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:14:11.724252888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:14:11.727193253Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED068F71CBCE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:14:14.534954818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:14:18.904014404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:14:18.908419392Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06AA324C3CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:14:19.53537377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-03T15:14:19.901381848Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BED06ADE58BB28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-03T15:14:22.111091626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:14:22.116039416Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06B6256EF11, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:14:22.611562405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:14:22.61899489Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06B804D65F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:14:22.957899772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a2f3450d55b25dc2 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"minio-operator\"}"
level=error ts=2026-07-03T15:14:22.962853389Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06B94D0428A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"minio-operator\"}"
level=info ts=2026-07-03T15:14:23.234547646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:14:23.237135548Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06BA5296D26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-03T15:14:24.535560976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:14:29.535493255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:14:30.085081758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:14:30.087770669Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06D3D7E0A92, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=info ts=2026-07-03T15:14:30.3809982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:14:30.383540213Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06D4F1F5882, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:14:31.262421437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:14:31.264925749Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06D83A8EBDE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:14:32.919469858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:14:32.922171467Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06DE66F6569, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:14:34.403356363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:14:34.406020472Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06E3EE0C4B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:14:34.535599791Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T15:14:34.53560428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:14:34.53562648Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:14:35.48186693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="171 B" avg_comp="171 B" total_uncomp="23 kB" avg_uncomp="23 kB" max_age=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:14:35.484691995Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06E7F2BCBC8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:14:37.180655413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:14:37.183398569Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06EE46C5869, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T15:14:37.637550072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:14:37.640248048Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06EFFA68FD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:14:39.535337288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:14:41.066430525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:14:41.07007357Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06FCC1520C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:14:41.473870692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:14:41.479862964Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED06FE4847138, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:14:42.802589715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:14:42.805225733Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07033834C1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-03T15:14:43.061754753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:14:43.06584465Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED070430B8484, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:14:44.53490232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:14:46.200277752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T15:14:46.205534285Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED070FE2D2BC0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-07-03T15:14:47.348723906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:14:47.351129248Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07142797971, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-03T15:14:48.335083264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:14:48.337682574Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0717D4572D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:14:48.94940214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:14:48.952466961Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED071A1EB004A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:14:48.964487738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:14:48.968641873Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED071A2E331C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:14:49.535669225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:14:50.643404731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:14:50.64604615Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07206DCDCCC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:14:51.757496074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a2f3450d55b25dc2 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"minio-operator\"}"
level=error ts=2026-07-03T15:14:51.760289154Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED072494697F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"minio-operator\"}"
level=info ts=2026-07-03T15:14:51.780348213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:14:51.782923225Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0724AA0E561, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-03T15:14:53.417667624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:14:53.42022613Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED072AC37D984, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T15:14:54.535875714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:14:59.535668635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:15:03.017488532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:15:03.020131651Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED074E86A4173, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-03T15:15:04.04301619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:15:04.045610926Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07525898126, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:15:04.515440304Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:15:04.535552586Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:15:04.535574827Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:15:04.535575735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:15:05.260749153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="486 kB" avg_uncomp="486 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-03T15:15:05.263538001Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0756E216784, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-03T15:15:05.825234005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:15:05.832142749Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0759005D59A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T15:15:09.535214182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:15:11.452884584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:15:11.458002261Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED076DF59C8B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-07-03T15:15:14.535758315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:15:15.364096865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:15:15.366742157Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED077C853CF41, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:15:15.366758748Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:15:15.366777365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:15:15.368774008Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED077C87604E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:15:18.187781759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:15:18.190385223Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07870A276C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:15:19.535279994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:15:19.873902641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:15:19.876731927Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED078D5258D8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:15:20.153477231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:15:20.156125787Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED078E5CC0042, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:15:20.156142314Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:15:20.156158896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:15:20.15821552Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED078E5EE4DB9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:15:22.213931408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:15:22.217506645Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07960AAC32B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:15:22.451020891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:15:22.457560201Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0796EF8C009, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-07-03T15:15:22.457587466Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:15:22.457617317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:15:22.459833363Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0796F1E5E79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-03T15:15:24.535460093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:15:25.26766792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e9ea4f05479baf45 immediate=false num_chunks=1 total_comp="101 kB" avg_comp="101 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:15:25.270606796Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07A16A534A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-epwq\"}"
level=error ts=2026-07-03T15:15:25.270623161Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:15:25.270644872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="190 B" avg_comp="190 B" total_uncomp="81 kB" avg_uncomp="81 kB" max_age=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:15:25.272905428Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07A16CB3A3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:15:27.312882582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:15:27.31547921Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07A908738E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:15:27.51090514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:15:27.513526906Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07A9C560425, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:15:28.913360243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:15:28.915939923Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07AEFECF835, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=warn ts=2026-07-03T15:15:29.535175164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:15:31.827885468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a2f3450d55b25dc2 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"minio-operator\"}"
level=error ts=2026-07-03T15:15:31.830602259Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07B9DA7B8F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"minio-operator\"}"
level=info ts=2026-07-03T15:15:31.94316107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="134 B" avg_comp="134 B" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" max_age=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:15:31.945906167Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07BA486F238, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:15:32.976100313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="638 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" max_age=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:15:32.980559384Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07BE231D3FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:15:34.120912648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T15:15:34.127344401Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07C268C9E27, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-07-03T15:15:34.127363744Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:15:34.127386152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="81 kB" avg_comp="81 kB" total_uncomp="641 kB" avg_uncomp="641 kB" max_age=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:15:34.12981017Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07C26B45925, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:15:34.317390429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:15:34.319872631Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07C3207B5AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:15:34.535842059Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:15:34.535870146Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:15:34.535871054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:15:34.917994347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:15:34.920955598Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07C55DAA861, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:15:34.987000623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:15:34.991536891Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07C5A106FE1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:15:36.359664585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:15:36.362378605Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07CABC55E2A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-07-03T15:15:36.362397241Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:15:36.362416101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="465 B" avg_comp="465 B" total_uncomp="230 kB" avg_uncomp="230 kB" max_age=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:15:36.36475563Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07CABEC2D88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:15:37.707775921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:15:37.710403553Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07CFC1EE66E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:15:37.822618744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:15:37.825097682Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07D02F53DE6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-03T15:15:39.535846525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:15:41.269382109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:15:41.271987535Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07DD068882A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:15:41.351703314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:15:41.354290135Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07DD5502FB4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T15:15:43.184892578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:15:43.187536389Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07E429559A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:15:44.25565748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:15:44.258177958Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07E82668DCA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:15:44.380548176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:15:44.38304238Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07E89D74F17, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-03T15:15:44.535730439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:15:44.921068581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:15:44.923601393Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07EAA0F3367, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:15:45.66653604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:15:45.669268662Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07ED680AB5F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:15:48.279246563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:15:48.281989056Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07F723C3015, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-07-03T15:15:49.535200336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:15:50.489389443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:15:50.49189735Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07FF5F56D42, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-07-03T15:15:50.491915202Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:15:50.491935593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="67 kB" avg_comp="67 kB" total_uncomp="582 kB" avg_uncomp="582 kB" max_age=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:15:50.494205505Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED07FF619A187, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:15:51.832885538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="486 kB" avg_uncomp="486 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-03T15:15:51.835800856Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED080460D763A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-03T15:15:53.432494806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:15:53.435174114Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED080A5632C54, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-03T15:15:54.535793089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:15:59.339843062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:15:59.342909511Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED082058277B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-03T15:15:59.535259299Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:16:01.022354668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:16:01.025016952Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08269C70996, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-07-03T15:16:01.560818508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:16:01.56371135Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08289E1E739, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:16:04.201734864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:16:04.208730321Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED083278B5D2F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:16:04.515596507Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:16:04.535779516Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T15:16:04.535789329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:16:04.535803866Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:16:09.535202962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:16:09.61849837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:16:09.623560545Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0846A4A1C78, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:16:10.041666793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:16:10.044269141Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED084835CBEB1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:16:10.85810237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:16:10.860732265Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED084B407A727, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:16:13.293161397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:16:13.296679066Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED085453891E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:16:13.321886771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:16:13.32652612Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08547015B1F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:16:13.565251837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:16:13.567867044Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08555633AD5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:16:13.567884347Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:16:13.567905888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:16:13.569926639Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED085558539AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:16:14.535473905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:16:15.442179723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a2f3450d55b25dc2 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"minio-operator\"}"
level=error ts=2026-07-03T15:16:15.444709502Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED085C541D6AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"minio-operator\"}"
level=info ts=2026-07-03T15:16:15.500758503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="638 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:16:15.504712014Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED085C8D59B6B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:16:16.269940263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:16:16.275009754Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED085F6BD3117, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T15:16:18.174069732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:16:18.176685785Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08668184BA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:16:18.630076542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:16:18.632697842Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0868346AC7A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-07-03T15:16:19.53531854Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:16:20.79266237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:16:20.795339363Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED087042E6310, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:16:22.636858687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:16:22.639664789Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED087721BB33D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:16:23.946470857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:16:23.949186051Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED087C02A36D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:16:24.409366524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:16:24.412058867Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED087DBC12367, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:16:24.412075647Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:16:24.412102017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:16:24.414211398Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED087DBE384FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:16:24.509093142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:16:24.511572509Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED087E1AFDCF6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:16:24.535827967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:16:24.606082388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:16:24.608652864Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED087E778D256, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:16:26.058134907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:16:26.060765742Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0883E064022, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:16:27.011578974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-03T15:16:27.014111622Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08876D933B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-07-03T15:16:27.014127831Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:16:27.014148018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="114 kB" avg_comp="114 kB" total_uncomp="786 kB" avg_uncomp="786 kB" max_age=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:16:27.018316496Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED088771B08F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-03T15:16:29.534904692Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:16:29.68721491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:16:29.690454225Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED089165C222D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:16:29.952507519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:16:29.958389361Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08926572EE6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:16:33.811593293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:16:33.81435227Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08A0C2BA03E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:16:34.535673583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:16:34.535678274Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:16:34.535706833Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:16:34.544169946Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-07-03T15:16:34.546081134Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010295888s
level=info ts=2026-07-03T15:16:35.084409889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:16:35.086908335Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08A5806DAF9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:16:37.4937776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:16:37.496667292Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08AE7A8294C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-03T15:16:39.535342212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:16:39.777788134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:16:39.780513646Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08B6FC9DDFE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-03T15:16:39.780531572Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:16:39.780552303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="118 B" avg_comp="118 B" total_uncomp="195 kB" avg_uncomp="195 kB" max_age=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:16:39.782840289Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08B6FEF02CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:16:40.706736843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:16:40.709605491Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08BA7287CE6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:16:40.70962292Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:16:40.709641281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:16:40.711640914Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08BA74C209A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:16:41.523214618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:16:41.525862294Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08BD7D19149, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:16:44.535711551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:16:45.501703792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:16:45.50441379Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08CC4F42D10, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-07-03T15:16:45.50443049Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:16:45.504451129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="158 B" avg_comp="158 B" total_uncomp="207 kB" avg_uncomp="207 kB" max_age=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:16:45.506734079Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08CC519E63C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-03T15:16:49.535275898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:16:50.312946207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="486 kB" avg_uncomp="486 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-03T15:16:50.315618931Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08DE3BA2BF5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-03T15:16:50.824594181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:16:50.832647673Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08E028B02EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T15:16:54.534941981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:16:55.8956849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:16:55.902613076Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08F30BA27AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:16:57.646545156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:16:57.649295354Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08F98D93FA4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:16:58.100379745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:16:58.103015359Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08FB3E35129, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:16:58.622560904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:16:58.625188743Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08FD303BC81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:16:59.186585526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:16:59.189516826Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08FF4A4939D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:16:59.251683713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="638 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:16:59.25584755Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08FF89BAACD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:16:59.343162811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:16:59.347749871Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED08FFE164CED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:16:59.499011062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a2f3450d55b25dc2 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"minio-operator\"}"
level=error ts=2026-07-03T15:16:59.501625833Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED090074214A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"minio-operator\"}"
level=warn ts=2026-07-03T15:16:59.53545018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:16:59.564587721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:16:59.567155691Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0900B2978EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:17:04.515493278Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:17:04.535264159Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-03T15:17:04.535297472Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:17:04.535322203Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:17:04.535331372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:17:04.535357719Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-07-03T15:17:04.535365443Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-03T15:17:04.535381546Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.622µs success=true
level=info ts=2026-07-03T15:17:05.173985444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:17:05.176928228Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0915987A272, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:17:06.362919236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:17:06.365666633Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED091A062C4EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:17:07.717649227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:17:07.72064274Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED091F12538DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-03T15:17:09.535711759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:17:09.844355179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:17:09.846944855Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0926FE25CCB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:17:12.719089087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:17:12.721892208Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0931B3F7F82, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:17:12.858091817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:17:12.861930651Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0932395079F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:17:12.95507255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:17:12.961037979Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED093297EE2A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-03T15:17:14.535352387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:17:14.708958677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:17:14.71368431Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED09391F6D92F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T15:17:15.230067862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:17:15.232676882Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED093B0E630E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:17:17.428707892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:17:17.436202296Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED09433F372B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:17:18.003751982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:17:18.006357552Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0945638C70F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:17:19.262609524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:17:19.265241808Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED094A1422642, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:17:19.535339188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:17:19.990725491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:17:19.993517946Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED094CCAB4A61, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:17:23.560153357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:17:23.562754993Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED095A169D554, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:17:23.589733653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:17:23.592118071Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED095A32A1943, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:17:23.592131457Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:17:23.59215164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:17:23.594127318Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED095A34A8B95, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T15:17:24.53549638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:17:24.60470436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:17:24.607322182Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED095DFABD336, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:17:27.697543534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:17:27.700467512Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED09698088841, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-03T15:17:29.535089235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:17:30.328747789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:17:30.331339343Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED09734D94979, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:17:34.535847698Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:17:34.535875325Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:17:34.535908728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:17:36.611555368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:17:36.614168452Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED098AB5564F8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:17:39.093116821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:17:39.100276214Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0993F8453AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:17:39.468248199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:17:39.474830078Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED09955D69411, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-07-03T15:17:39.535313151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:17:39.737687985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="638 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:17:39.741758941Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED09965C0B420, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:17:40.675599902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:17:40.68029229Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0999DB1F316, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T15:17:44.535631889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:17:44.612923511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:17:44.615798577Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED09A88436099, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:17:44.743945825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="486 kB" avg_uncomp="486 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-03T15:17:44.746599308Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED09A9010DD3A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-03T15:17:49.535190068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:17:50.248077923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:17:50.250721933Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED09BD822DE7D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:17:50.762008049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a2f3450d55b25dc2 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"minio-operator\"}"
level=error ts=2026-07-03T15:17:50.764590533Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED09BF6C3CE48, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"minio-operator\"}"
level=info ts=2026-07-03T15:17:52.378661474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:17:52.381612251Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED09C57253A29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:17:52.493866381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:17:52.496495Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED09C5DFDD295, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:17:54.095946451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:17:54.098656134Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED09CBD7CC93D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-03T15:17:54.53584495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:17:58.357499914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:17:58.360163904Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED09DBB7F0EBA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:17:58.861498688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:17:58.864188589Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED09DD989B0E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-03T15:17:59.535222341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:18:00.175689713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:18:00.178319692Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED09E27DE158B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:18:02.227541659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:18:02.230461888Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED09EA22E66BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:18:03.021868596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:18:03.024538905Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED09ED18417AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:18:03.123876395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:18:03.126406786Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED09ED7955338, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T15:18:04.515424009Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-03T15:18:04.534880238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:18:04.534883386Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:18:04.534918249Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:18:04.583347752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:18:04.586124997Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED09F2E96E425, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:18:07.55354903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:18:07.556361493Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED09FDFA2F40D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:18:08.136151429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:18:08.140570545Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A002754F93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:18:09.115629826Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BED0A03C94D385, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-03T15:18:09.535876511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:18:10.175080191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:18:10.178476772Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A07BEC041B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:18:10.39195465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:18:10.396244503Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A088E737CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:18:10.396261063Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:18:10.396281417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:18:10.398276738Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A0890918EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:18:10.728449127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:18:10.731040044Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A09CDC1D0C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-03T15:18:14.535654421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:18:15.063371025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:18:15.066014037Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A19F3DE7A6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:18:16.73693728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:18:16.739550637Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A202FEA574, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:18:16.739567304Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:18:16.739586334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:18:16.741658814Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A2032195BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:18:19.101893597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:18:19.108810467Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A2903783BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T15:18:19.535676334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:18:24.161476437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:18:24.167900658Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A3BDC23A07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-03T15:18:24.535830768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:18:25.260183941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:18:25.264575602Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A3FF1F880E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:18:25.360895176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:18:25.365403701Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A40521B3EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:18:26.376311145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:18:26.378827534Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A4418A4852, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:18:26.745073144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:18:26.74751232Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A457848F5C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:18:27.272742302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:18:27.275215729Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A476F8822A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:18:28.555800591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:18:28.558446724Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A4C374C163, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:18:28.846098587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:18:28.848639593Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A4D4C09BD6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:18:29.534904245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:18:30.953007413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:18:30.955639243Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A552567039, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:18:34.535616794Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:18:34.535645547Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:18:34.535645105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:18:36.100207487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:18:36.102855716Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A68522C391, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:18:37.403193752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a2f3450d55b25dc2 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"minio-operator\"}"
level=error ts=2026-07-03T15:18:37.40580094Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A6D2CC36E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"minio-operator\"}"
level=info ts=2026-07-03T15:18:38.228160536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="486 kB" avg_uncomp="486 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-03T15:18:38.230797673Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A703F8ED6F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-03T15:18:38.988991103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:18:38.991572222Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A73151B694, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T15:18:39.535724606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:18:40.534634686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:18:40.537436571Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A78D75025C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:18:41.840384437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:18:41.843235862Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A7DB49B38D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-07-03T15:18:44.535392214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:18:47.149288981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:18:47.1520759Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A917B96EA4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T15:18:48.835014923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:18:48.837809714Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A97C30FA2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:18:48.841844647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:18:48.844362143Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A97C98CF8C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-07-03T15:18:48.84437411Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:18:48.844392512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:18:48.850068387Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A97CEFBC57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-03T15:18:49.535626986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:18:49.960432469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:18:49.965157802Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0A9BF64F556, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:18:54.534961014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:18:56.321285458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:18:56.324019245Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0AB3A69B8DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:18:56.44620924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:18:56.44878208Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0AB41D942B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:18:57.17726368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:18:57.179814808Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0AB6D6C7D92, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:18:57.487500891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:18:57.490095453Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0AB7FE9B6F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-03T15:18:59.535882719Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:19:00.660013713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:19:00.662764819Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0AC3D058366, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:19:01.996581187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:19:01.999305435Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0AC8CAF295F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:19:02.809873633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:19:02.812740869Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0ACBD296DC2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:19:02.980064832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:19:02.982524592Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0ACC74AD2A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:19:03.444583268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:19:03.454045583Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0ACE362A2AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:19:03.45407575Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:19:03.454098632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:19:03.462846305Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0ACE3EB5354, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:19:04.515466092Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:19:04.535378654Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:19:04.535401118Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:19:04.53548812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:19:04.561744847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:19:04.566548982Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0AD25B2FB91, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:19:04.717003479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:19:04.722112914Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0AD2EFA0FBB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:19:04.722129337Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:19:04.722151221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:19:04.726585303Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0AD2F415419, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:19:08.599649519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:19:08.606171128Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0AE167AD387, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:19:09.347241691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:19:09.351854139Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0AE42EDE6DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:19:09.420998848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:19:09.423535983Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0AE4734BE39, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-07-03T15:19:09.535002897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:19:14.534886964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:19:14.918911942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:19:14.925884916Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0AF8F2B07DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:19:15.071094843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:19:15.075619032Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0AF9817D2AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:19:16.201941462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:19:16.204664849Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0AFDB640E9F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:19:16.204681868Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:19:16.204701794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:19:16.206726105Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0AFDB8626A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:19:17.861556114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:19:17.864219706Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B03E4EDC69, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:19:18.825552938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:19:18.828986992Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B077CFE46B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-07-03T15:19:19.535015673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:19:19.691007595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="486 kB" avg_uncomp="486 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-03T15:19:19.695922531Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B0AB7BD4D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-03T15:19:19.695937774Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:19:19.695957217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="176 kB" avg_comp="176 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" max_age=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:19:19.698693019Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B0ABA5E588, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:19:22.34953272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:19:22.352509803Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B149D4A8FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:19:24.535653851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:19:25.001350001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:19:25.004374549Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B1E7E5076C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:19:26.8403368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:19:26.845096875Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B2559C3A87, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:19:27.942493677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:19:27.945210364Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B2972E0F03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:19:27.945230718Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:19:27.945250781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:19:27.947391738Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B2975262EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:19:28.270041714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:19:28.27258243Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B2AAB2AAEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:19:29.535715707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:19:32.288776105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:19:32.29181554Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B39A40E0C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:19:34.535336733Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T15:19:34.535337975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:19:34.535365084Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:19:34.806087204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:19:34.808842451Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B4304890B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:19:35.936314139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a2f3450d55b25dc2 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"minio-operator\"}"
level=error ts=2026-07-03T15:19:35.938901518Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B473A53F52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"minio-operator\"}"
level=info ts=2026-07-03T15:19:36.276217061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:19:36.283004284Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B488277B24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:19:39.063250685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:19:39.069591804Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B52E3F1093, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:19:39.535594535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:19:39.5827455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:19:39.585723412Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B54D01EBBA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:19:39.834906957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:19:39.838352287Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B55C11316A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:19:41.25412289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:19:41.258708782Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B5B0BAAE18, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:19:41.452348439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:19:41.454987723Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B5BC6D6CA5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:19:43.607301905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:19:43.609962538Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B63CDF508B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-03T15:19:44.53526778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:19:45.910565144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:19:45.913181196Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B6C627FB81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:19:48.078430064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:19:48.080829442Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B7475CF90A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T15:19:49.535021366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:19:49.913235245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:19:49.915774083Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B7B4BAC7C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:19:50.221857392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:19:50.224329259Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B7C71F76BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:19:50.811578588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:19:50.814380819Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B7EA4A11E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:19:51.566813106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:19:51.569578246Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B8174D9A1F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:19:53.70293197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:19:53.705664067Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B896A04659, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-03T15:19:54.535240947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:19:57.571365729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:19:57.574117232Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B97D334437, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:19:57.800652544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:19:57.80454126Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B98AF0957A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:19:57.804557767Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:19:57.804579737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:19:57.810488492Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B98B4D72F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:19:57.907734057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:19:57.912102795Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0B99159BDE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T15:19:59.53583699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:20:01.063860787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:20:01.066754362Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0BA4D60F555, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:20:01.066771119Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:20:01.066790547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:20:01.069091448Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0BA4D85EDB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:20:01.88699521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:20:01.889703149Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0BA7E6E775A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:20:04.516274473Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-03T15:20:04.535721151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:20:04.535722361Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:20:04.535749381Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:20:09.535661788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:20:09.726599778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:20:09.731749868Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0BC51B74BD5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:20:12.475239794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:20:12.477870668Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0BCF589ABAB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:20:12.925938242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:20:12.928611582Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0BD106745FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:20:13.736362007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:20:13.738896914Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0BD40B369AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:20:14.315346186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:20:14.317943776Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0BD63368205, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-03T15:20:14.535027504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:20:15.518400839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:20:15.521314657Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0BDAAEFB175, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:20:17.041118079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:20:17.043778002Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0BE05AFAC94, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:20:17.756887645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:20:17.760913907Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0BE306E512E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:20:18.165378014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a2f3450d55b25dc2 immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"minio-operator\"}"
level=error ts=2026-07-03T15:20:18.171578702Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0BE48E855E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"minio-operator\"}"
level=error ts=2026-07-03T15:20:18.171597504Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:20:18.171618497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" max_age=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:20:18.17684463Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0BE49385034, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-03T15:20:19.53578002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:20:20.118125648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:20:20.125145281Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0BEBD592D9B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:20:20.445839857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:20:20.450922738Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0BED0C48E1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:20:21.052787449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:20:21.058211065Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0BEF4F64A1A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-07-03T15:20:24.534984426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:20:27.341689703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:20:27.346613546Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C06BC782A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:20:27.462805569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:20:27.465287729Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C072DB239D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-03T15:20:29.53586116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:20:31.029349265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:20:31.032218801Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C14774DC26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:20:31.954535315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:20:31.957271647Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C17E996502, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:20:33.934000819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:20:33.936827297Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C1F495AB60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:20:34.535470387Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:20:34.535498637Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:20:34.535512005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:20:34.847617673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:20:34.850211742Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C22B0770FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:20:35.43065664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:20:35.433143491Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C24DC77230, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:20:37.386841442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:20:37.389486793Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C2C261B1FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T15:20:38.328509095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:20:38.33180375Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C2FA8D5167, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T15:20:39.535590228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:20:41.759649156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:20:41.762524424Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C3C707DAEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:20:42.710566098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:20:42.713369395Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C3FFB57320, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:20:42.767358427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:20:42.769851082Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C40313AA58, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:20:43.448714812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:20:43.45147091Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C42BB45CCC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:20:44.119503874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:20:44.122489604Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C453B2DB7D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-03T15:20:44.53581243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:20:45.481630335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:20:45.484256799Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C4A4DE5046, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:20:47.037318644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:20:47.040263284Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C5019C285F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-03T15:20:49.535303728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:20:49.948005304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:20:49.950648787Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C5AF169232, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:20:53.202552356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:20:53.208221988Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C6712E982F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:20:54.535366441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:20:55.82286286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:20:55.827759851Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C70D64134E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:20:58.346762416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:20:58.349492938Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C7A3B2CD44, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-03T15:20:59.535047417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:20:59.876678882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:20:59.879563336Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C7FEE4F09E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:21:03.149860358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:21:03.152849062Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C8C1FE979A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:21:04.287688269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:21:04.290285375Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C905CC2641, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:21:04.515422663Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:21:04.535306192Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:21:04.535330413Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:21:04.535343118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:21:05.170502905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:21:05.177386247Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0C93AABD6F0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:21:09.386073227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:21:09.391677918Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0CA35DD9CB7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T15:21:09.535822187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:21:12.578233722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:21:12.581214312Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0CAF3F6E6EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-07-03T15:21:12.581234508Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:21:12.581254092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:21:12.5833045Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0CAF41B8DD0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:21:12.657545582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:21:12.660195729Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0CAF8AF270B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:21:13.049420879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:21:13.052111503Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0CB100B0DA0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-07-03T15:21:14.535347872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:21:16.536417329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:21:16.539155567Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0CBDFE26568, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:21:17.832619833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:21:17.83536782Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0CC2D2482C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-03T15:21:18.28374873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:21:18.286701006Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0CC480CB91F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:21:18.546791866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:21:18.552549609Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0CC57E59ECA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-07-03T15:21:19.535244519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:21:24.535722747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:21:24.681075108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:21:24.685898147Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0CDC578518E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:21:27.139486488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:21:27.142110134Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0CE57DFF793, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:21:27.361658204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:21:27.36432595Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0CE651DF08E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:21:28.687852037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:21:28.690557556Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0CEB42B4B70, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T15:21:29.535767581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:21:30.143014387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:21:30.145736325Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0CF0AE678D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:21:31.610630112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:21:31.613602656Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0CF62647B54, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:21:34.021785288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:21:34.024561721Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0CFF218EE36, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:21:34.151918147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:21:34.15452879Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0CFF9D86726, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:21:34.535397491Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:21:34.535429918Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:21:34.535424298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:21:34.544947902Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-07-03T15:21:34.546888415Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011397559s
level=info ts=2026-07-03T15:21:34.798072662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:21:34.802397942Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D020743949, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:21:35.279606334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:21:35.285809136Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D03D467E8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:21:36.364149727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:21:36.366809729Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D07DB46A3C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:21:38.224633193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:21:38.227310374Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D0EC99E4DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:21:39.316440848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:21:39.318956144Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D12DAB8115, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:21:39.318993929Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:21:39.319020655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:21:39.321099041Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D12DCDCA82, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-07-03T15:21:39.535714093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:21:40.639949538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:21:40.64277033Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D17C9253CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:21:42.364554465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:21:42.367768103Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D1E362C497, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:21:42.972506965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:21:42.975262901Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D2079973EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:21:44.535891627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:21:46.12409559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:21:46.127157976Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D2C3760047, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:21:48.237127528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:21:48.23986484Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D3416531A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-03T15:21:49.535484722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:21:51.028090345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:21:51.031278784Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D3E7BE3A48, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-03T15:21:54.535685121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:21:54.820192057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:21:54.822877705Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D4C9C6166E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:21:56.893224397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:21:56.89594011Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D54555919D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:21:56.895956199Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:21:56.895992695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:21:56.898079885Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D5457979B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:21:57.291275405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:21:57.293954978Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D55D0F1757, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:21:57.391334124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:21:57.39819509Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D56345E702, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T15:21:59.535825099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:22:03.814012792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:22:03.820712576Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D6E21602EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:22:03.820732416Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:22:03.820754116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:22:03.823027258Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D6E23B5B86, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:22:04.515485147Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:22:04.534984633Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:22:04.534996835Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-03T15:22:04.535006581Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:22:04.535060356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:22:04.53514018Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-07-03T15:22:04.535149481Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-03T15:22:04.535169284Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.751µs success=true
level=info ts=2026-07-03T15:22:05.086012352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:22:05.089291299Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D72DB07BD5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:22:08.484722926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:22:08.489753458Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D7F86279D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T15:22:09.535469812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:22:09.5796254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:22:09.582222386Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D8397FC705, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:22:12.032519059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="75 kB" avg_uncomp="75 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-03T15:22:12.035160747Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D8CBB5183D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-07-03T15:22:12.035176693Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:22:12.035194787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="124 B" avg_comp="124 B" total_uncomp="48 kB" avg_uncomp="48 kB" max_age=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:22:12.0374122Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D8CBD7AD0D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:22:12.75416239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:22:12.757673827Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D8F6C59E9C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:22:13.518906095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:22:13.523913035Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D92471BBFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:22:14.407245381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:22:14.409927529Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D95940464D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T15:22:14.535625787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:22:14.825081183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:22:14.827709062Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D9722756FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:22:14.881753315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:22:14.884263041Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0D975873422, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-03T15:22:19.535476921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:22:21.556352789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:22:21.558917954Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DB035EB555, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:22:21.558932932Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:22:21.55895162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:22:21.560993756Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DB037FCB97, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:22:23.614006304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:22:23.616540154Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DB7E03C2EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-07-03T15:22:24.535004865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:22:25.429853157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:22:25.432716654Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DBEA42C31A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:22:25.932289273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:22:25.934953363Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DC083347C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:22:25.934986557Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:22:25.935004679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:22:25.93704364Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DC0854AE53, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:22:26.038393729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:22:26.040666245Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DC0E80E4BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:22:26.040678624Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:22:26.23520207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:22:26.23771004Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DC1A3E96C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:22:26.760287887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:22:26.763406059Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DC39911741, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:22:26.836529091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:22:26.839415757Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DC3E1A8C96, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:22:28.210160506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:22:28.212801898Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DC8FF7866C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:22:28.214835422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:22:28.21707604Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DC903B4055, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:22:28.217088876Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:22:28.21710619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:22:28.219242865Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DC905DE700, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:22:28.34965167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:22:28.352069434Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DC98453937, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:22:28.943704548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:22:28.947763069Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DCBBC6FAA4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:22:29.534998867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:22:29.910068008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:22:29.91606705Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DCF57D39C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:22:32.456176025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:22:32.458918297Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DD8D0E9FFE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:22:33.258122779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:22:33.260811762Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DDBCD9D7E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:22:33.677010107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:22:33.679634865Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DDD5D1BD2B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:22:33.679649526Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:22:33.679667716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:22:33.681774945Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DDD5F49542, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:22:34.534908786Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:22:34.534940747Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:22:34.534956818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-03T15:22:37.09671354Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BED0DEA180C59E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-03T15:22:39.330775393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:22:39.333798669Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DF26D29F65, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:22:39.41898344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:22:39.42166013Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DF2C11CC09, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:22:39.535651666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:22:39.644992928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:22:39.647583002Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DF398810A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:22:40.350864224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:22:40.353622146Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DF639DA362, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:22:40.73035877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:22:40.733007819Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0DF7A3A6EDC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:22:43.037565532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:22:43.040476133Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E003C318AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-03T15:22:44.535007708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:22:44.558184848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:22:44.56076746Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E05E61DB6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:22:47.372542156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:22:47.375397827Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E10624C861, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:22:47.95422624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:22:47.957031303Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E128CF4232, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:22:48.891902196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:22:48.899844423Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E1610270C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T15:22:49.53591661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:22:53.019290802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:22:53.025460532Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E256EBD0E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:22:53.295980079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:22:53.300481916Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E2674F4DD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:22:54.201572049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:22:54.204909381Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E29D3926F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-07-03T15:22:54.53586371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:22:54.902298789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:22:54.906875073Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E2C70FFF81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:22:56.61306379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:22:56.615695324Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E32CE9AE80, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:22:57.108082979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:22:57.110502723Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E34A68CAD9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:22:57.110519019Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:22:57.110538922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:22:57.11253593Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E34A88CEFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T15:22:59.535359129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:23:00.175284409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:23:00.177886553Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E4013CAF9E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:23:00.390515833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:23:00.393047595Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E40E0F52D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:23:00.733282842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:23:00.735695935Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E4227CBB05, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:23:00.735711141Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:23:00.735729298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:23:00.737706055Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E4229D4514, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:23:02.900112191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:23:02.90287052Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E4A3A87231, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:23:04.516225565Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:23:04.53557459Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:23:04.535596256Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:23:04.53560122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:23:06.268244716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:23:06.270938114Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E56C6879CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:23:07.999820936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:23:08.002505329Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E5D39EAFA9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T15:23:09.535763134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:23:10.363880546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:23:10.36656438Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E66087B5C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:23:10.366581854Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:23:10.36660393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:23:10.368638375Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E660A95232, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:23:11.083530947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:23:11.086225016Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E68B6CF322, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:23:11.560731571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:23:11.563519139Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E6A7DF910E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:23:11.618730842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:23:11.621457115Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E6AB533813, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:23:14.146692983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:23:14.149436413Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E742004CE3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:23:14.535772894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:23:14.752514313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:23:14.755351366Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E7661E41D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:23:18.80041666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:23:18.803379564Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E85766B2E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-03T15:23:19.53530921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:23:19.83090857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:23:19.833946433Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E894D2940E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:23:21.586010738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:23:21.588944015Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E8FD6C9EBF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:23:24.401812314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:23:24.404632381Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E9A5435BB0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:23:24.404653059Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:23:24.404671607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:23:24.40690697Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E9A5683528, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-03T15:23:24.535221578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:23:25.339718914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:23:25.34230401Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0E9DD270384, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:23:27.058489664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:23:27.061065423Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0EA4398DDDD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:23:28.395727304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:23:28.399812662Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0EA936527DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:23:29.535219128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:23:30.007624797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:23:30.013742268Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0EAF396EC05, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:23:30.013762858Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:23:30.013783621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:23:30.015807754Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0EAF3BA0A1A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:23:33.306262403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:23:33.308917665Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0EBB7FFEB64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:23:34.535080181Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:23:34.535108022Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:23:34.535123028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:23:35.21015169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:23:35.212679717Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0EC297A1243, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:23:36.69456337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:23:36.697811415Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0EC81FC9A12, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:23:37.311308407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:23:37.313803586Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0ECA6B5E78B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-03T15:23:39.535689187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:23:42.443114495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:23:42.445982287Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0EDD89AD238, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:23:43.759695182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:23:43.762459997Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0EE271372D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:23:44.408223373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:23:44.411392798Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0EE4DC1E55F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:23:44.479576093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:23:44.492177564Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0EE5292F3AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T15:23:44.535859126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:23:45.902618374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:23:45.90742999Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0EEA6ED0C4E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:23:46.728867137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:23:46.731779722Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0EED81136D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:23:46.731800717Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:23:46.731821499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:23:46.733902544Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0EED83414AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:23:47.701010359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:23:47.703599741Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0EF11FD4096, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:23:48.54176867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:23:48.544512162Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0EF441CD373, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:23:48.868302041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:23:48.871098832Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0EF57932F24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-03T15:23:49.534899694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:23:50.114637991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:23:50.117227739Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0EFA1DAEE82, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:23:52.031446017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:23:52.034255466Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0F0141D3E82, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:23:52.620022036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:23:52.623512609Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0F0373D838B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:23:53.669151956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:23:53.673693675Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0F075D5A3D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-03T15:23:54.535284232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:23:55.405358065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:23:55.408051202Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0F0DD355E53, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:23:56.14231233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:23:56.145003395Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0F1092214FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:23:58.43948974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:23:58.442108434Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0F1920E60B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T15:23:59.535296366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:23:59.762600658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:23:59.765497148Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0F1E0EE3DBE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:24:02.439176593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:24:02.442144552Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0F28078357A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:24:04.515414528Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:24:04.53587658Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:24:04.535899263Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:24:04.535918394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:24:05.05168624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:24:05.054630015Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0F31C302FD8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:24:06.034162682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:24:06.036730543Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0F356BA8A13, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:24:06.720399637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:24:06.72308919Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0F37FA3340D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-03T15:24:09.535496318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:24:11.050349597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:24:11.054478582Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0F481CFA09B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:24:12.33037205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:24:12.335802489Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0F4CE2F1519, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-07-03T15:24:14.535468141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:24:15.306230859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:24:15.309189787Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0F57F67D9B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:24:15.888742513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:24:15.891601235Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0F5A220715B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-03T15:24:19.53527403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:24:21.057751496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:24:21.060429513Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0F6D635CE2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:24:23.338687656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:24:23.341395403Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0F75E2A8F71, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-03T15:24:24.535232583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:24:29.535658036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:24:29.599784344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:24:29.602553447Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0F8D35C3A53, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:24:30.298933154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:24:30.301567634Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0F8FD0649E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:24:31.856535504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:24:31.859250344Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0F959DE7FE9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=warn ts=2026-07-03T15:24:34.535399914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:24:34.535410581Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:24:34.535437825Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:24:34.879864673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:24:34.886864043Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0FA0E5475A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:24:36.333015957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:24:36.337689386Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0FA64CD8D5D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:24:36.575879329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:24:36.578463622Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0FA73286C80, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:24:37.014883502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:24:37.01761473Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0FA8D54DD51, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:24:38.280451257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:24:38.282935436Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0FAD8C15567, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:24:39.303946149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:24:39.306642342Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0FB15C4D287, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-03T15:24:39.535406131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:24:39.570503013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:24:39.573480854Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0FB25ACA332, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:24:39.97459486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:24:39.979361867Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0FB3DDC565E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:24:40.604319363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:24:40.607037082Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0FB63470AF8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:24:40.952879723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:24:40.955472425Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0FB780B4A25, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:24:43.939746078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:24:43.942491573Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0FC2A16198F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:24:43.942508232Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:24:43.942528174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:24:43.944573047Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0FC2A388AF5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:24:44.535222893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:24:45.042571921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:24:45.045262429Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0FC6BD00939, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:24:45.045281108Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:24:45.045301867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:24:45.047366628Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0FC6BF42276, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:24:45.332481771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:24:45.335243827Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0FC7D195363, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:24:47.839552563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:24:47.843142613Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0FD12930ACE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:24:48.632018066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:24:48.636389586Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0FD41DDAFAD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:24:49.535073319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:24:53.049739171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:24:53.0540525Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0FE492D9A9C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:24:53.534375842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:24:53.539059276Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0FE66165A00, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T15:24:54.535632429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:24:55.442411592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:24:55.445108875Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0FED7B1F18A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:24:58.272423581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:24:58.275142223Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED0FF8061546C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:24:59.535597834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:25:00.906255593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:25:00.908914575Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1001D5D5965, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:25:01.031277558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:25:01.03418182Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10024D3CAC5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:25:04.286907748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:25:04.290259742Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED100E6E6359E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:25:04.515434921Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:25:04.535014181Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T15:25:04.535017594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:25:04.535040332Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:25:04.792008449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:25:04.794658607Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10104F9599D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:25:06.419476009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:25:06.422156984Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10165FA2A7E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:25:06.808408344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:25:06.810986594Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1017D2837E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:25:09.53520371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:25:12.66061968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:25:12.66360854Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED102D9FE8255, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:25:12.804019036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:25:12.806788457Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED102E287AE4B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:25:13.61116661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:25:13.613953334Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10312A3DDB3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-03T15:25:14.535466778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:25:19.535027507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:25:20.65414024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:25:20.657161034Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED104B67279D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-07-03T15:25:20.657182182Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:25:20.65720234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:25:20.661921644Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED104B6BE8275, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:25:20.717350378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:25:20.721306492Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED104BA47E8FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:25:21.836318244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:25:21.838859823Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED104FCE3A355, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:25:24.18224779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:25:24.184884252Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10588B8E9CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-03T15:25:24.535619624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:25:24.69001257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:25:24.692695489Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED105A6FD4A45, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:25:25.887310638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:25:25.895248771Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED105EEAA3E65, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:25:25.895270255Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:25:25.89529658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:25:25.905105329Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED105EF41CE11, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:25:26.944204477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:25:26.948764001Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1062D764439, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:25:27.575375872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:25:27.577887317Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10652F6EAD3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T15:25:29.535037613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:25:31.807589068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:25:31.811229337Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1074F48AC48, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:25:31.811250406Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:25:31.81127369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:25:31.816239738Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1074F9792AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:25:32.536371102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:25:32.541269908Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1077ACC2367, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:25:32.841005601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:25:32.843689261Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1078CD39029, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:25:33.578859529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:25:33.581675731Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED107B8D03D16, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:25:34.383851269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:25:34.386532192Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED107E8C9DFCB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-07-03T15:25:34.534907187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:25:34.534925551Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:25:34.534952407Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:25:34.739809599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:25:34.742443995Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED107FDFF4878, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:25:36.247495752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:25:36.254503972Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10858206E15, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:25:36.519440937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:25:36.525870018Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED108684D2184, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:25:36.785629802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:25:36.790739312Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10878169B36, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:25:37.529826796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:25:37.534774571Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED108A46EF9CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:25:39.535929208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:25:39.55306333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:25:39.555855976Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1091CE6A703, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:25:42.239018554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:25:42.241994039Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED109BD014FA8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:25:44.535438681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:25:44.980684747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:25:44.984036283Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10A6072A8D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:25:46.950087836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:25:46.955157791Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10AD5EEAEFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:25:46.983175977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:25:46.985579717Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10AD7C089FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:25:46.985592109Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:25:46.985608019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:25:46.987596822Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10AD7E0D5A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:25:49.354960375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:25:49.35760487Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10B65221D07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-07-03T15:25:49.53493338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:25:51.135658723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:25:51.138264036Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10BCF44A515, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:25:52.669098123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:25:52.672245034Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10C2AB1122A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:25:52.672265304Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:25:52.672285858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:25:52.674759917Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10C2AD9D859, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-07-03T15:25:54.535804229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:25:59.413823642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:25:59.421702156Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10DBCFFEE78, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T15:25:59.535234714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:26:00.702150912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:26:00.708119544Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10E09A9F453, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:26:00.772288242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:26:00.774702975Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10E0DA56E2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:26:01.035463761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:26:01.037913215Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10E1D55E320, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:26:01.202347617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:26:01.204931874Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10E274A0CFB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:26:04.516431153Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-03T15:26:04.535616816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:26:04.535630547Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:26:04.535650189Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:26:04.747420035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:26:04.750006455Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10EFA978C4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:26:07.848739701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:26:07.851754267Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10FB376B756, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:26:08.009242405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:26:08.011805086Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED10FBD0272D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:26:09.45676742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:26:09.459386765Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1101349E740, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:26:09.459404075Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:26:09.459422807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:26:09.461457918Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED110136CEC5B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=warn ts=2026-07-03T15:26:09.535894854Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:26:10.044896183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:26:10.048087466Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED110365F402C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:26:11.692536611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:26:11.698225661Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11098BC1FF5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-07-03T15:26:14.534925853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:26:15.28507192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:26:15.287746105Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1116EAFF18E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:26:15.465283628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:26:15.468403008Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED111796F7E8A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:26:17.634006985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:26:17.63672445Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED111FAB33100, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:26:18.82827644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:26:18.83186584Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11241EE3DF8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:26:18.871892169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:26:18.87655698Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11244994F19, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:26:19.056088294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:26:19.060087834Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1124F89AA9D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:26:19.060103076Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:26:19.060122597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:26:19.065726623Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1124FE1F333, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:26:19.178150973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:26:19.183047132Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11256DE7E27, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T15:26:19.535876881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:26:20.94684258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:26:20.949567092Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED112C02924D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:26:20.962544809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:26:20.964912769Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED112C115466A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-03T15:26:24.534930372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:26:27.370740906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:26:27.37346178Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1143F0DB5BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:26:28.383297576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:26:28.385890064Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1147B65E894, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:26:29.027663032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:26:29.030390249Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED114A1D1052A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:26:29.231862358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:26:29.234582435Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED114ADFC04DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-07-03T15:26:29.53522755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:26:30.187467239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:26:30.190365788Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED114E6F39323, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:26:30.734319073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:26:30.738313821Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED115079D7094, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:26:31.181327712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:26:31.187917298Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1152267662E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:26:31.187936178Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:26:31.187956792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:26:31.190546847Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1152292128A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:26:31.324388182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:26:31.327086877Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1152AB3363E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:26:34.535508932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:26:34.53551864Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:26:34.535544914Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:26:34.543997993Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-07-03T15:26:34.545785101Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010452096s
level=info ts=2026-07-03T15:26:38.133831026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:26:38.136760666Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED116C0976249, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-07-03T15:26:39.535578701Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:26:42.134934849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:26:42.13777999Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED117AF13D40E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-07-03T15:26:44.534879092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:26:45.917240148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:26:45.92036531Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1189086E1BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:26:46.476663648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:26:46.479274942Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED118B1D93A28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:26:47.517316162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:26:47.520033536Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED118EFE0DD50, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-03T15:26:49.535666394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:26:52.237540173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:26:52.240294069Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11A093C4552, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:26:54.146359906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:26:54.153230313Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11A7B40988A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T15:26:54.535101265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:26:57.253813686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:26:57.259948598Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11B346D9AA9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:26:57.259986084Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:26:57.260005974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:26:57.262276032Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11B3492F341, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:26:57.936442086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:26:57.939149041Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11B5CE8C749, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:26:58.975909509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:26:58.978529821Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11B9ADCF6D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:26:59.534906979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:27:00.197778636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:27:00.200630503Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11BE3B420B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:27:00.636793717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:27:00.643782425Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11BFE1E6DCF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:27:01.446302913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:27:01.448957115Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11C2E1BFE5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:27:02.42091592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:27:02.42404179Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11C683AE009, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:27:03.791451005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:27:03.796203333Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11CBA045510, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:27:04.51559912Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:27:04.534618641Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-03T15:27:04.534736208Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-07-03T15:27:04.534743999Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-03T15:27:04.534760465Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.253µs success=true
level=warn ts=2026-07-03T15:27:04.53493531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:27:04.534950711Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:27:04.53498106Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:27:04.908921553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:27:04.911922862Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11CFC82BB57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:27:07.27708217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:27:07.281272824Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11D89BE3695, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:27:07.992419173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:27:07.997136735Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11DB4699256, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:27:09.24072297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:27:09.243735378Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11DFEB64164, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-03T15:27:09.535371832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:27:10.543889751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:27:10.546714229Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11E4C60E726, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:27:10.62924322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:27:10.631914894Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11E51757901, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:27:13.261768362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:27:13.26451796Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11EEE5F28F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-03T15:27:14.535059493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:27:14.621347677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:27:14.624036728Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11F3F671CAE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:27:14.954706204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:27:14.958128228Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11F53523CCB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:27:15.199374124Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BED11F61B6417F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-03T15:27:16.524581542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:27:16.529742391Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED11FB0FDF60A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:27:19.131211626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:27:19.133861924Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1204C3673C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-07-03T15:27:19.535610941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:27:20.390552238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:27:20.393183095Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1209745E3C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:27:24.270095303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:27:24.273092691Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1217E86F96E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-03T15:27:24.53523773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:27:25.648839019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:27:25.651513033Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED121D0B24999, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:27:29.535242024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:27:32.24301441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:27:32.245953046Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED12359C0E2E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:27:33.071283758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:27:33.074272048Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1238B1B71B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:27:34.535351099Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:27:34.535385538Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:27:34.535399332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:27:37.675187306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:27:37.677889134Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1249D85B509, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-07-03T15:27:39.534959405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:27:39.765573825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:27:39.768296675Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1251A1F3991, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:27:40.499905473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:27:40.502748648Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED12545E53FA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:27:43.046075827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:27:43.048743736Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED125DDA67CAF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T15:27:44.535299835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:27:45.734240002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:27:45.736737979Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1267DDEB9B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:27:46.705407854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:27:46.708209195Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED126B7C4EDE5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:27:49.535356816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:27:50.813453044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:27:50.816105278Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED127AC9E7FA4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:27:50.972346418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:27:50.975049319Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED127B616ED43, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:27:51.91507483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:27:51.922112071Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED127EE8C0C07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:27:53.422424986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:27:53.428907238Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED128485BCCC2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:27:53.528361853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:27:53.532294956Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1284E85A889, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:27:53.671516598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:27:53.675758309Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED12857130A9D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:27:54.086859563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:27:54.089415417Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1286FBA2F07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:27:54.47463757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:27:54.477776831Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED12886DD66CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-03T15:27:54.53490987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:27:57.403677671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:27:57.408713235Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1293591D87C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:27:59.294637821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:27:59.29737272Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED129A62525A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-03T15:27:59.535931107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:28:00.544804409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:28:00.547400424Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED129F0A76991, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:28:04.51542637Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:28:04.5349898Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:28:04.535022032Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:28:04.535056769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:28:04.752755392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:28:04.756196699Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED12AEB83E761, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:28:06.8378918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:28:06.842631177Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED12B67E0BB8C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:28:07.117515795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:28:07.120212812Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED12B786C21BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T15:28:09.53539064Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:28:10.936503862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:28:10.939322373Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED12C5C0D24CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:28:12.892232466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:28:12.89477631Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED12CD09D088D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:28:13.057384301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:28:13.059758778Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED12CDA73341F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-07-03T15:28:13.059771639Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:28:13.059788907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:28:13.061743147Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED12CDA9343DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-03T15:28:14.535211885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:28:15.468436517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:28:15.470949727Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED12D6A2AE1BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:28:18.599907805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:28:18.602747363Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED12E24D506CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:28:19.143298006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:28:19.145956782Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED12E4537182F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:28:19.145990152Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:28:19.146008432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:28:19.148122218Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED12E4559AD29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-03T15:28:19.535180638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:28:24.434251951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:28:24.436930742Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED12F8093E98E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:28:24.47890701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:28:24.482295773Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED12F83454475, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-07-03T15:28:24.535636152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:28:29.240085221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:28:29.24281224Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1309F07D434, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-03T15:28:29.535518024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:28:30.014000611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:28:30.016581086Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED130CD26D89A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:28:30.016599657Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:28:30.016618012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:28:30.018658606Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED130CD49AD87, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:28:30.631745056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:28:30.634387858Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED130F1FA7CEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:28:30.634404968Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:28:30.634424496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:28:30.636436857Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED130F21C410D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:28:32.622176264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:28:32.624851002Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED131689DEAD0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:28:33.102009288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:28:33.10464509Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13185372F1E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:28:33.104663595Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:28:33.104684782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:28:33.106740158Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED131855A1286, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:28:34.276870226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:28:34.280313301Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED131CB49C73A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:28:34.535576873Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:28:34.535609315Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:28:34.535614683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:28:35.531096899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:28:35.53381163Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13216019EC5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:28:35.533826155Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:28:35.533845063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:28:35.535993324Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1321624A50D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:28:36.214646852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:28:36.217164072Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1323EBCC5B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-03T15:28:39.53500136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:28:39.825257949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:28:39.828299044Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13315F92D08, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:28:40.990452686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:28:40.99676479Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1335B9F9728, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:28:41.022918552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:28:41.025344805Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1335D540790, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:28:41.025356967Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:28:41.025373547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:28:41.027405796Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1335D756755, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:28:41.767003421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:28:41.769853866Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13389B2A5D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:28:42.166000514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:28:42.168745166Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED133A17ACCED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:28:43.831132122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:28:43.834031294Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13404BB6EAE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:28:44.299189957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:28:44.301761269Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED134209DE311, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:28:44.30177898Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:28:44.301797654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:28:44.303938071Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13420C100B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-03T15:28:44.535391286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:28:44.871790742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:28:44.875814387Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13442D49BEE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:28:45.992334832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:28:45.996999774Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13485A8154C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:28:46.313894456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:28:46.3206806Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13498F3E002, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:28:47.365559711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:28:47.370271777Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED134D781C6E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:28:47.909773253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:28:47.912265028Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED134F7D1F188, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:28:48.893593185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:28:48.896328229Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1353277C1CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:28:49.490266741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:28:49.492797605Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1355606CCC5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=warn ts=2026-07-03T15:28:49.535068704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:28:53.657561283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:28:53.660210856Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1364E6BB304, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:28:53.660227622Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:28:53.660246289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:28:53.662231149Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1364E8DABB5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-07-03T15:28:54.535724892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:28:54.85800588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:28:54.861411849Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED136960476BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:28:55.518696482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:28:55.523143519Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED136BD75C532, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:28:56.538135136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:28:56.540675658Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED136FA1CAACE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:28:57.746951179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:28:57.7496292Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED137422BCBB5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:28:59.381081519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:28:59.383897464Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED137A393E21D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-03T15:28:59.535196587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:29:04.108536806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:29:04.111205303Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED138BD599440, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:29:04.515732616Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:29:04.534844671Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:29:04.534868416Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:29:04.534882261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:29:05.100005264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:29:05.102595901Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED138F871CF9E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:29:05.80827558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:29:05.810787362Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13922A77F92, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:29:07.608925989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:29:07.611588991Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1398DFC856F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:29:09.249186899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:29:09.252020523Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED139EFC2BC38, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:29:09.535361974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:29:10.373891286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:29:10.376639508Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13A32CC6A8C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:29:10.753280872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:29:10.756205081Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13A496AFBAD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:29:11.50763281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:29:11.510689069Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13A7662F4F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:29:12.178560091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:29:12.181275452Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13A9E5CBCFB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:29:12.181297831Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:29:12.181316828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:29:12.183391517Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13A9E80026A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:29:13.430259012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:29:13.43299303Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13AE8F8A23B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-07-03T15:29:14.535121589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:29:16.842034853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:29:16.844714755Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13BB453CB09, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-03T15:29:19.534986054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:29:19.831470226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:29:19.834153536Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13C66821BC5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:29:22.493202292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:29:22.495878161Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13D05290DBC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:29:22.495897323Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:29:22.495918772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:29:22.498068191Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13D054CA89C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:29:23.836490986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:29:23.839119146Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13D55396818, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:29:24.53529101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:29:25.509348438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:29:25.512047866Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13DB8F0446E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:29:26.853203712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:29:26.855849618Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13E09096295, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:29:28.026471203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:29:28.02920432Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13E4EF954D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:29:29.144308869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:29:29.152013431Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13E91E3ECA0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T15:29:29.535679484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:29:30.983695612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:29:30.98850902Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13EFF5AF62B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:29:31.777454242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:29:31.780820975Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13F2E938FEF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:29:32.44366177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:29:32.450590944Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED13F567E7210, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:29:34.535003926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:29:34.535033721Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:29:34.535052247Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:29:35.8159992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:29:35.820372139Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1401F5C0E46, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:29:36.272409931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:29:36.275722058Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1403A81B3E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:29:36.743151693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:29:36.74779437Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14056A47584, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:29:36.747813425Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:29:36.747833686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:29:36.749961695Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14056C7D942, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T15:29:39.535468102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:29:39.613769421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:29:39.616857542Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14101A5682F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:29:40.668798892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:29:40.671580396Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14140843B4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:29:41.49064506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:29:41.493212795Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED141717D0DC3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-07-03T15:29:44.535064311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:29:46.388326864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:29:46.39081763Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED142956990D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:29:46.390833414Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:29:46.390851309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:29:46.392885587Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED142958AA94E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:29:47.703270889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:29:47.705958107Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED142E3CC727C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:29:49.310585481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:29:49.313235771Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14343991C37, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:29:49.323263678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:29:49.325640775Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14344583B31, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:29:49.325652632Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:29:49.3256688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:29:49.327848377Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED143447A7093, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-03T15:29:49.534922084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:29:50.377517799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:29:50.380142025Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED143833061C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:29:51.082830482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:29:51.085388108Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED143AD39D8BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:29:52.453184306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:29:52.456035551Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED143FEEC9AC8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:29:53.08577314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:29:53.088847869Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14424A34E7A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-07-03T15:29:54.535114819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:29:55.37990447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:29:55.382576767Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED144AD5C3EFB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:29:57.426433631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:29:57.429191478Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1452757E479, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:29:57.528614235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:29:57.530940547Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1452D6AAB2F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:29:57.530953216Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:29:57.530986274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:29:57.533109582Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1452D8BAE5B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:29:59.080579133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:29:59.083220216Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14589EF3750, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:29:59.535915934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:29:59.627540962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:29:59.630170589Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED145AA892CF5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:30:03.281417787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:30:03.283930208Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1468451D734, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:30:04.485293786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:30:04.487838494Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED146CC13CC01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:30:04.515889288Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-03T15:30:04.53553005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:30:04.535531022Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:30:04.535553924Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:30:05.635007871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:30:05.637770359Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED147109D1B93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:30:06.488123769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:30:06.490563951Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1474372D2F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T15:30:09.535211053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:30:09.674540439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:30:09.677305219Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED148016206BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:30:10.504569049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:30:10.507429267Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14832DCFB21, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:30:12.802845619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:30:12.805954823Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED148BBDE0549, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-03T15:30:14.534897027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:30:16.138431943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:30:16.14342924Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14982CC1851, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:30:16.893636949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:30:16.896317563Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED149AFAD7B58, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:30:17.35567033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:30:17.35846083Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED149CB380D7C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-03T15:30:19.535099871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:30:24.445158053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:30:24.447804581Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14B71C7E957, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:30:24.533005995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-03T15:30:24.535174233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-03T15:30:24.535615232Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14B77049DC7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:30:26.06385599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:30:26.067125931Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14BD24C4505, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:30:26.479297054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:30:26.483574926Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14BEB1F0F81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:30:26.767645683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:30:26.773598686Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14BFC68FD9D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:30:26.81078962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:30:26.817711603Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14BFF0922F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:30:29.063456907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:30:29.069096086Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14C853B384C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-07-03T15:30:29.535170537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:30:29.678489193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:30:29.683336717Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14CA9D6FB94, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:30:32.425982279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:30:32.429102579Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14D4D7FF3FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:30:34.535441722Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:30:34.535477106Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:30:34.535487143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:30:38.808471468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:30:38.811211082Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14EC9E79956, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-07-03T15:30:39.535554893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:30:40.877485003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:30:40.880021194Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14F4538B292, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:30:40.880041582Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:30:40.880060749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:30:40.882064755Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14F4559F004, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:30:41.548610665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:30:41.551475456Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14F6D3CA6B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:30:43.789956455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:30:43.792524429Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED14FF2D0A898, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:30:44.074191468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:30:44.076841621Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED15003C2CB58, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:30:44.535018963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:30:44.597171491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:30:44.599662241Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED15022EE1803, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:30:44.656748532Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BED15026573F25, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-03T15:30:47.129144606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:30:47.131843262Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED150B9DBD42E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:30:49.07523679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:30:49.077943867Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1512DDA976E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-03T15:30:49.53594776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:30:50.212335479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:30:50.215133718Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED15171A2592D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:30:50.501690386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:30:50.504314435Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED15182DF935D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:30:53.253160603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:30:53.255865054Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED15226E09D50, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T15:30:54.535950329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:30:55.214895661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:30:55.217885926Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1529BD13E23, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:30:57.369160631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:30:57.371836743Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1531C368CE5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:30:58.181006793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:30:58.183594803Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1534C9942BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:30:58.968949463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:30:58.971590021Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1537B90A66F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T15:30:59.53583486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:31:01.452893515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:31:01.455469472Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1540F9E1D6D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:31:01.455486737Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:31:01.455503908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:31:01.457569389Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1540FBFDCC1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:31:04.14795675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:31:04.150709565Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED154B0435060, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:31:04.515765908Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:31:04.535300054Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:31:04.535324105Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:31:04.535316686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:31:07.555007946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:31:07.559151353Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1557B5B166D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:31:09.097205709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:31:09.099865778Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED155D7418EFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-03T15:31:09.53560001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:31:10.524807289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:31:10.527913761Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1562C5F2558, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:31:11.734168008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:31:11.740382952Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED15674A3954D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:31:12.174553456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:31:12.179542126Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1568ED03F81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:31:12.799793218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:31:12.802561286Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED156B3F44908, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:31:13.146207809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:31:13.14893639Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED156C898A5DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T15:31:14.535686107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:31:16.453722692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:31:16.457954701Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1578DD45BE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:31:17.682083082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:31:17.686611209Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED157D70FB5D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:31:18.029921845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:31:18.032685254Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED157EBB0FE51, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:31:18.242922834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:31:18.251070712Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED157F8B5ED28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T15:31:19.535094027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:31:22.905860965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:31:22.910607713Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1590E70A776, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T15:31:24.535600314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:31:25.655579076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:31:25.658198547Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED159B2360CBA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:31:26.165626744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:31:26.168373243Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED159D09D44EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:31:27.661516756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:31:27.664571623Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED15A29CAFF03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-07-03T15:31:29.535125784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:31:30.359667696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:31:30.362712147Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED15ACA9D9E9F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-03T15:31:34.535060452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:31:34.535070413Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:31:34.535099259Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:31:34.544352528Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-07-03T15:31:34.546268673Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011394038s
level=info ts=2026-07-03T15:31:34.793278685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:31:34.796040524Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED15BD2DB8DC1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:31:36.842684079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:31:36.845442762Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED15C4D04C812, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:31:39.112520478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:31:39.115178895Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED15CD44D5DF2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:31:39.115196822Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:31:39.115216348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:31:39.117260624Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED15CD4711BBC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:31:39.535371715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:31:39.90166279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:31:39.904453988Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED15D03599634, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:31:42.871616181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:31:42.874362903Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED15DB45EDAD0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:31:44.535833756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:31:47.191492562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:31:47.194128464Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED15EB5D956F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:31:48.058339868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:31:48.061149814Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED15EE987026B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:31:48.100145202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:31:48.102752094Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED15EEC02EAC9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:31:49.342847534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:31:49.34556919Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED15F3616A3D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T15:31:49.535741694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:31:49.64109345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:31:49.64376656Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED15F47DC2537, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:31:50.12278818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:31:50.125505902Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED15F6492619A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:31:51.650097301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:31:51.652804643Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED15FBF9BFAF0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-03T15:31:54.534908513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:31:55.302597314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:31:55.305513397Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1609951CAB6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:31:55.940358215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:31:55.942980725Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED160BF51C4B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:31:56.317958319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:31:56.320819094Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED160D5D7E7E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:31:59.261424299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:31:59.26510448Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1618554EE99, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:31:59.265123433Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:31:59.26514319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:31:59.271764364Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16185BC777C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:31:59.477267421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:31:59.481744844Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED161923F37F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T15:31:59.535106272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:32:03.478084089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:32:03.48060661Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED162809A0658, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:32:04.029323255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:32:04.036433341Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED162A1BAFC71, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:32:04.03645422Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:32:04.036478446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:32:04.045577817Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED162A246DD1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:32:04.131944609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:32:04.13650044Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED162A7B1FB25, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:32:04.515405588Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:32:04.535215498Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:32:04.535239266Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:32:04.535234167Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-07-03T15:32:04.535267173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:32:04.535333831Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-07-03T15:32:04.535340953Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-03T15:32:04.53535808Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.319µs success=true
level=info ts=2026-07-03T15:32:06.611388802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:32:06.614187731Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1633B604799, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:32:07.034574244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:32:07.037113879Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1635495D3F8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:32:09.535139902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:32:10.05171469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:32:10.054821116Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16408736D03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-07-03T15:32:10.054838873Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:32:10.054858548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:32:10.059713005Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16408C0199E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:32:12.363007676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:32:12.369078152Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16492646CC7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:32:12.552661905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:32:12.55761192Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1649DA119A6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:32:13.588409304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:32:13.590993826Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED164DB39C2FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:32:13.952977529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:32:13.957060379Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED164F10B5C72, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:32:14.535768591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:32:16.586609616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:32:16.592108599Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1658E1968D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:32:16.592129853Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:32:16.592149428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:32:16.594827508Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1658E42A7D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:32:18.196012917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:32:18.203000276Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED165EE1ECEFF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T15:32:19.534953905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:32:20.340087672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:32:20.345176418Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1666DCC19E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:32:21.821945097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:32:21.825081045Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED166C6016E09, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-03T15:32:24.535836053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:32:26.2478578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:32:26.252800096Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED167CDECCCD2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:32:28.212106272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:32:28.214723593Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16842DD436E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-03T15:32:29.535825444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:32:30.397531861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:32:30.400222003Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED168C520880B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:32:31.448536542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:32:31.45113372Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16903C4BCF9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:32:31.498161133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:32:31.500648717Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16906B995C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:32:31.950141781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:32:31.952700894Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16921AB8EB4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:32:32.842103183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:32:32.844743325Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16956D686EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:32:33.032224207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:32:33.034911632Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED169622C3556, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:32:33.674623857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:32:33.677333218Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1698875F767, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:32:34.279902267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:32:34.282671178Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED169AC8B0CE8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:32:34.535595379Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:32:34.535623224Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:32:34.535656784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:32:35.781526193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:32:35.784538747Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16A060F4813, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:32:36.868347112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:32:36.870954259Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16A46D12C6E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:32:36.870984499Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:32:36.871003106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:32:36.873207421Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16A46F5D6CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:32:37.088659834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:32:37.092242868Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16A53FF64D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:32:38.489203172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:32:38.495910795Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16AA7A946E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-03T15:32:39.534939434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:32:41.468185435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:32:41.476422375Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16B59527A7D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:32:44.466134381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:32:44.471160149Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16C0BD222A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-03T15:32:44.535671394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:32:48.375752808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:32:48.378582505Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16CF4B9A4EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:32:48.768732582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:32:48.771538717Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16D0C254A5B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-03T15:32:49.535915447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:32:52.216432221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:32:52.219138769Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16DD9A3A9BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:32:54.235417198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:32:54.238100107Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16E51FA0BEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:32:54.535052803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:32:58.178224634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:32:58.180910672Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16F3CFCEF93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:32:58.180927878Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:32:58.180948317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:32:58.182962365Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16F3D1F9183, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:32:58.378497002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:32:58.381107298Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16F48EC0864, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:32:58.82933645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:32:58.832060401Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16F63CD4D9F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-03T15:32:59.535349783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:32:59.732688467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:32:59.735654485Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16F99A964DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:33:00.624197599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:33:00.629135944Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED16FCEE9B2A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:33:04.515460856Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-03T15:33:04.535513943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:33:04.535527013Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:33:04.535550047Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:33:07.535225606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:33:07.53929826Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1716ACC3B5E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:33:07.539319442Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:33:07.539339656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:33:07.544741167Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1716B1FF64D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:33:08.366818268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:33:08.371863153Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1719C6A0CBF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:33:09.013906148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:33:09.016625603Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED171C2D8B023, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:33:09.286212033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:33:09.28880101Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED171D3125EAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-03T15:33:09.53539239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:33:11.441688137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:33:11.444486531Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED172538E9813, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:33:12.264778877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:33:12.267994549Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED17284A2BDE8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:33:12.268014226Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:33:12.268036998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:33:12.270624706Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED17284CD1781, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:33:14.293276569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:33:14.295940277Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED172FD84B1EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=warn ts=2026-07-03T15:33:14.535856315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:33:18.288415196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:33:18.291121944Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED173EBA75B21, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T15:33:19.535236209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:33:19.771712739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:33:19.774426905Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED174440FF8BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:33:20.267007729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:33:20.269819881Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED17461970554, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:33:21.348272516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:33:21.351198398Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED174A20C88AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:33:21.497437155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:33:21.500115191Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED174AAEC5DF7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:33:22.490977363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:33:22.495465614Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED174E6402CF9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:33:23.984723739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:33:23.99110953Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1753F65CC0F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:33:24.35416779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:33:24.361709154Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED175557D805C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T15:33:24.535679404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:33:26.809353299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:33:26.813918359Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED175E7A79428, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:33:28.514562625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:33:28.51767661Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1764D33D0B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-03T15:33:29.535117731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:33:29.856666552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:33:29.86055718Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1769D3D82AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:33:30.835248336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:33:30.837885246Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED176D77F8F72, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:33:31.2789417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:33:31.281595434Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED176F1F34886, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:33:31.491016101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:33:31.496896337Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED176FEC824E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:33:33.932983143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:33:33.93646829Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED17790307DDC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:33:34.534958846Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:33:34.535009517Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:33:34.535076585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:33:35.81861066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:33:35.823466997Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED17800A8C72C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:33:39.535393229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:33:39.594552294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:33:39.597285615Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED178E1999072, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:33:39.597300292Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:33:39.597318869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:33:39.599415675Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED178E1BCECBF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-03T15:33:44.535226938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:33:45.798513954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:33:45.801483942Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED17A5364C8C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:33:48.05560588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:33:48.058635419Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED17AD9EF9A0B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-03T15:33:49.535512254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:33:50.07200449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:33:50.077946245Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED17B524C166C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:33:51.398854446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:33:51.401471179Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED17BA12FB347, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:33:51.524006785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:33:51.527701376Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED17BA8B78D50, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:33:51.646935891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:33:51.651463867Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED17BB016198C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:33:52.410492097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:33:52.413232715Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED17BDD7E2D15, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:33:53.222425999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:33:53.225306293Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED17C0DE4D8DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:33:54.177983788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:33:54.180577898Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED17C46D5F5B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-03T15:33:54.53489962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:33:54.587066067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:33:54.58961564Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED17C5F3770D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-07-03T15:33:59.535544809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:34:00.562550136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:34:00.565691562Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED17DC3695755, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:34:01.684389358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:34:01.687047521Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED17E0640EA24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:34:04.515880988Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-03T15:34:04.535001616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:34:04.535012018Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:34:04.535035129Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:34:06.068251069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:34:06.070910866Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED17F0B8D8DFE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-03T15:34:09.535185731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:34:09.972346552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:34:09.975074648Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED17FF441346E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:34:10.155806753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:34:10.162792295Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED17FFF7310B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:34:11.929154848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:34:11.935851318Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18069216B2D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:34:12.966210235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:34:12.968923003Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED180A6B41EF0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:34:13.302643197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:34:13.305278637Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED180BAC14FA0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T15:34:14.535451055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:34:14.800071776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:34:14.801142421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:34:14.802703096Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18114024D8F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:34:14.803622292Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18114125502, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:34:17.04471988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:34:17.047380174Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18199CC35AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:34:19.535376172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:34:19.546461663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:34:19.549423989Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1822EED55CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:34:19.615557416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:34:19.618143096Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18233076AC9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:34:19.778715127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:34:19.781173351Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1823CBF9833, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:34:20.294135253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:34:20.296800615Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1825B7A87D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:34:20.878898793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:34:20.882215236Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1827E6022B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:34:22.465422715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:34:22.468131624Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED182DCE6355A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:34:23.152248644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:34:23.154826137Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18305D50C17, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-03T15:34:24.534884783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:34:29.535863918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:34:34.535568977Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:34:34.535599893Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:34:34.535646679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-03T15:34:35.857280606Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BED185FAF7C5AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-03T15:34:36.072178056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:34:36.076198578Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18608017B5E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:34:36.585065625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:34:36.589111098Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18626946A8F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:34:37.008218789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:34:37.012875543Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1863FD746E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:34:37.012894793Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:34:37.012917328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:34:37.015009838Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1863FF959EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:34:37.294877868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:34:37.297303039Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18650CADB3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:34:37.953286697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:34:37.955892118Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED186780BD0E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:34:39.047711599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:34:39.05073435Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED186B94CD000, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:34:39.095716175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:34:39.098432531Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED186BC26CBB0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-03T15:34:39.535877868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:34:41.871212475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:34:41.876304626Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18761B81372, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-07-03T15:34:44.535483766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:34:47.102011739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:34:47.104868059Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED188995D7BCD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:34:48.339493103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:34:48.342113845Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED188E31D4C80, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:34:48.949767267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:34:48.952291188Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED189077C6DAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-03T15:34:49.53503443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:34:51.308577473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:34:51.315761793Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18994587DD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T15:34:54.535211626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:34:58.012844665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:34:58.018867764Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18B23E498C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:34:58.817079765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:34:58.819603058Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18B539F4EB0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:34:58.819621562Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:34:58.819641448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:34:58.821629511Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18B53C0BE61, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T15:34:59.535581651Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:35:00.967743006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:35:00.970486679Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18BD3D1FD25, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:35:00.970504686Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:35:00.970522575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:35:00.972799798Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18BD3F88672, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:35:01.281616738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:35:01.284293791Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18BE686A5D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:35:01.284314822Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:35:01.284336586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:35:01.286538355Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18BE6ABEB96, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:35:03.318879194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:35:03.321619006Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18C5FF5D046, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:35:04.515463707Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-03T15:35:04.535571935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:35:04.535584187Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:35:04.535603533Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:35:04.815103273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:35:04.817673921Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18CB92274A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:35:04.888735718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:35:04.89116443Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18CBD842D1D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:35:05.067005301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:35:05.069606185Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18CC8260E5E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-07-03T15:35:05.069620748Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:35:05.069637107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:35:05.071505552Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18CC846ED22, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:35:06.054088848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:35:06.056527594Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18D02FADCDB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:35:06.478898076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:35:06.481407634Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18D1C4D14F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:35:06.481427687Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:35:06.481448475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:35:06.483490237Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18D1C6F17B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:35:06.944983612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:35:06.947457823Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18D38152F57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:35:06.947475785Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:35:06.947494628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:35:06.949518291Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18D3836506A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:35:07.196184331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:35:07.198634116Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18D470DE942, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:35:07.539497823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:35:07.541915556Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18D5B849836, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:35:07.541934194Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:35:07.541952124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:35:07.544008595Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18D5BA516BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:35:07.859631578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:35:07.862224929Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18D6E9989D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:35:09.458820359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:35:09.461380913Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18DCDEBB154, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-03T15:35:09.53535324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:35:14.535451523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:35:16.810091864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:35:16.81266881Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18F8417B74F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:35:16.846701097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:35:16.849090193Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18F8644268D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:35:17.38850384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:35:17.391225825Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18FA69313E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:35:17.991009299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:35:17.993633998Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18FCA7AEAD3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:35:18.864856146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:35:18.868279951Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED18FFE9D06BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-07-03T15:35:19.535534928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:35:19.660873704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:35:19.66669592Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1902E347ADC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:35:20.256585522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:35:20.259273526Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1905185DA1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:35:20.262302411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:35:20.264574062Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19051D8B77A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:35:22.845900397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:35:22.848996205Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED190EBE08490, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:35:23.185907575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:35:23.18857352Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19100204CB4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-03T15:35:24.535924135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:35:26.65006993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:35:26.65255155Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED191CE9A0FAA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:35:26.652568332Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:35:26.652587851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:35:26.65451345Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED191CEB932C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:35:27.45563177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:35:27.45810764Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED191FE9D215B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T15:35:29.535365836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:35:29.577405533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:35:29.580385603Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1927D1A0B53, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:35:31.268503312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:35:31.271255882Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED192E1E34B81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:35:32.654434859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:35:32.657244039Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED193347FC915, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:35:33.043369063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:35:33.047635837Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1934BC5167B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:35:34.535679698Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:35:34.535710643Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:35:34.535764096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:35:38.430629241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:35:38.435475372Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1948CE94792, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:35:38.439492851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:35:38.442167244Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1948D4F6612, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-03T15:35:39.535349913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:35:40.906013554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:35:40.910681297Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1952071B8AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:35:42.898683574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:35:42.901422281Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1959718F5C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-03T15:35:44.535091546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:35:44.899531079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:35:44.902138894Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1960E59E73C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:35:46.248157194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:35:46.250843719Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1965EBD909F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:35:49.535219572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:35:49.925432838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:35:49.932900959Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1973A3670F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:35:52.287942954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:35:52.294396418Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED197C6F5EA40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:35:54.350018524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:35:54.352640265Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19841A64218, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:35:54.534879054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:35:54.552006351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:35:54.554554764Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1984DAF3B71, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:35:54.554568072Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:35:54.554586824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:35:54.556626813Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1984DD12924, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:35:55.010824551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:35:55.010833278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:35:55.013451453Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1986908A81B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:35:55.014351387Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1986918C39D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:35:55.014365686Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:35:55.014383041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:35:55.016305676Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19869378603, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:35:55.275177469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:35:55.277637671Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19878C8BFBF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:35:55.455095679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:35:55.457704369Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19883837D21, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:35:55.975349601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:35:55.977925145Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED198A28597EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:35:55.977939447Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:35:55.977956099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:35:55.980035061Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED198A2A7E3B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:35:56.130540386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:35:56.13352059Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED198ABC9D3F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:35:56.64000119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:35:56.642494598Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED198CA226272, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:35:59.53564693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:36:02.390558929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:36:02.394185261Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19A20F56CB1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:36:03.166312071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:36:03.169068218Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19A4F2526BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:36:04.516285178Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-03T15:36:04.535901478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:36:04.535925211Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:36:04.535945346Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:36:05.269439964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:36:05.272273064Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19ACC810AB4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:36:07.386846212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:36:07.389730807Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19B4AB78B10, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-03T15:36:09.53547955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:36:09.626784445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:36:09.62956593Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19BD038614F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:36:09.667606739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:36:09.670153358Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19BD2A512C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:36:11.357867673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:36:11.360913633Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19C37695A63, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:36:11.360930946Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:36:11.360952524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:36:11.36328626Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19C3791682C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:36:12.969717296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:36:12.972421863Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19C977886BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:36:13.930147054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:36:13.932783821Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19CD0B7DE1E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T15:36:14.535082337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:36:16.618558594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:36:16.621679095Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19D70FAFBC1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:36:16.773086023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:36:16.775630556Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19D7A2A5173, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:36:17.280906743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:36:17.283574594Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19D986F70B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-07-03T15:36:19.535757439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:36:22.932201998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:36:22.935337404Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19EE94CF8AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-03T15:36:24.535855536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:36:25.444223779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:36:25.447342178Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19F7F089481, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:36:25.597916545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:36:25.602285026Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19F8845019C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:36:25.602300174Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:36:25.602324394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:36:25.60429854Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19F8866A5D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:36:25.98981314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:36:25.992404871Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED19F9F859617, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T15:36:29.535032511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:36:31.626815593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:36:31.631003554Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A0EF9C4986, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:36:33.511687356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:36:33.516738776Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A16002CBB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:36:33.516758312Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:36:33.516778637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:36:33.518927204Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A16026195A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:36:34.535689492Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:36:34.535717299Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:36:34.535714632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:36:34.540716852Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-07-03T15:36:34.542689091Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007206756s
level=info ts=2026-07-03T15:36:35.046994433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:36:35.049597724Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A1BB6008F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:36:36.680761154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:36:36.683491451Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A21CC43E99, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:36:37.697390142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:36:37.700120931Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A2595B466F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:36:37.736420741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:36:37.738848085Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A25BABF882, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:36:38.183933806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:36:38.191995579Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A276ADC4CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:36:38.268283373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:36:38.27270853Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A27B7C13A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:36:38.570073749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:36:38.572658249Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A28D5E2FD6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:36:39.535838619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:36:39.888407792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:36:39.891255133Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A2DBF54C61, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:36:40.793365545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:36:40.795994632Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A311E31B89, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:36:41.147946897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:36:41.150494954Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A32705422C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:36:41.9848249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:36:41.987561982Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A358E97D1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-03T15:36:44.535276522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:36:47.138504392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:36:47.141248138Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A48C17CFD7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:36:49.333440166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:36:49.336314266Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A50EED0DD9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-03T15:36:49.535524572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:36:50.732531927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:36:50.735285173Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A5624FB2F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:36:52.161926219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:36:52.164541655Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A5B7824D9E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:36:53.018465868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:36:53.021942732Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A5EA9C4507, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:36:53.253480919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:36:53.259675811Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A5F8C76C5F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-03T15:36:54.535225891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:36:55.715670792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:36:55.718341023Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A68B52F27F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:36:56.238618174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:36:56.241879335Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A6AA8201E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-03T15:36:59.535876439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:37:04.515675816Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:37:04.534609225Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-03T15:37:04.534795778Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-07-03T15:37:04.534804978Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-03T15:37:04.534825464Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.812µs success=true
level=info ts=2026-07-03T15:37:04.534901089Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:37:04.534924306Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:37:04.534922082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:37:06.778571812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:37:06.782963413Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A91ED1D24A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:37:07.43127921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:37:07.43392893Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A945A25712, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:37:09.205002752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:37:09.207668864Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A9AF5B2084, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:37:09.258700717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:37:09.261173891Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A9B28BEC9B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:37:09.261187079Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:37:09.261204503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:37:09.263734963Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A9B2B56CA5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T15:37:09.535113672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:37:09.597232887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:37:09.600234263Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1A9C6BEAE79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:37:11.285687863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:37:11.288381142Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1AA2B5FE157, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T15:37:14.535082895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:37:19.518858346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:37:19.521839459Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1AC16213762, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-03T15:37:19.535353269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:37:19.839961014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:37:19.846361837Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1AC2978AC99, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:37:21.17815956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:37:21.180768085Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1AC7901E67E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:37:21.668461844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:37:21.675619097Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1AC9680A912, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:37:23.202761319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:37:23.207658966Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1ACF1D1FA69, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-03T15:37:24.535752691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:37:24.858203854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:37:24.860858035Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1AD545AABD5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:37:25.649070848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:37:25.6518143Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1AD837F269F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:37:26.182153553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:37:26.184938749Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1ADA345DB2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:37:26.377399907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:37:26.380155749Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1ADAEE8F620, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:37:27.818009498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:37:27.822159009Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1AE04DCF4F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:37:29.03143634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:37:29.036540407Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1AE4D3E8F57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:37:29.535191861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:37:30.356153836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:37:30.358777308Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1AE9C0E5FCD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:37:33.193534375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:37:33.196806227Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1AF45398237, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:37:34.171930495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:37:34.176996546Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1AF7FA469DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:37:34.534848246Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:37:34.534878491Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:37:34.534904282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:37:36.419575054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:37:36.422279066Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B00578ABB4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:37:37.105287904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:37:37.107917224Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B02E5698E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:37:37.107935732Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:37:37.107954752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:37:37.110155781Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B02E79EB5C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:37:37.12426524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:37:37.126510107Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B02F74C2A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-03T15:37:39.535674092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:37:43.117544213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:37:43.120586069Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B194B7E4C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:37:43.877859831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:37:43.880676132Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B1C206C338, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-03T15:37:44.535486143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:37:46.360979924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:37:46.363760199Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B25606EE60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:37:48.231130706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:37:48.2338441Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B2C57F1CE1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:37:49.122771959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:37:49.125541131Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B2FAA56C84, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:37:49.535729769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:37:52.297413972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:37:52.300396569Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B3B7E0EF3C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:37:54.379993348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:37:54.382797302Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B43400878B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T15:37:54.535476771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:37:54.92892157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:37:54.931529294Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B454B4747A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:37:57.301543893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:37:57.304218955Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B4E22148E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T15:37:59.535437052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:38:01.638550242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:38:01.641382313Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B5E4A44FD7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:38:02.261081192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:38:02.932077963Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B6318F626B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:38:02.932111967Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:38:02.932143249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:38:02.942059045Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B6322A3F19, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:38:03.679013288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:38:03.683667998Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B65E600B0A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:38:04.515408809Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:38:04.535141297Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:38:04.535168518Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:38:04.535186183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:38:08.372470178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:38:08.375189053Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B77602580B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:38:08.377218529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:38:08.379848626Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B776483748, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-07-03T15:38:09.535078304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:38:09.667410832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:38:09.670520465Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B7C3395A32, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:38:10.234149593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:38:10.24051091Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B7E5325AC8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:38:10.656359762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:38:10.660948318Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B7FE41A444, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:38:10.747268901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:38:10.749725193Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B8038D11C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-03T15:38:14.535810837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:38:14.581954682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:38:14.585399932Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B8E82ADECB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:38:17.843522581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:38:17.845938436Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B9AA842564, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:38:18.594449711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:38:18.602211002Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B9D7969F1E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:38:19.259897476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:38:19.264794423Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1B9FF14EE53, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:38:19.53582455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:38:19.89140317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:38:19.894083952Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1BA2496A146, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:38:20.939287798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:38:20.942142397Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1BA630D0754, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:38:23.188685474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:38:23.191807586Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1BAE9256693, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:38:24.133524444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:38:24.136581564Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1BB2174AF27, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:38:24.508645739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:38:24.512006033Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1BB37D7D0F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:38:24.512025513Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:38:24.512046962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:38:24.51737927Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1BB382B1D28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-07-03T15:38:24.535585682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:38:25.415998148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:38:25.420433658Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1BB6DFD14D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:38:26.317894777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:38:26.320779387Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1BBA3A7D59E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:38:28.306311809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:38:28.309099641Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1BC1A29B2CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-03T15:38:29.535589879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:38:30.882882057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:38:30.885697083Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1BCB3BDEBAE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-03T15:38:34.535579571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:38:34.535620325Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:38:34.535646688Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:38:34.793329444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:38:34.796099456Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1BD9CD1C955, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:38:37.255415839Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BED1BE2F6B82C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-03T15:38:37.373876143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:38:37.376586133Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1BE36A14569, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:38:37.376602943Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:38:37.37662239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:38:37.378804229Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1BE36C50D4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:38:39.535476508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:38:41.361879637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:38:41.365392688Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1BF2460F733, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:38:42.350228484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:38:42.355110947Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1BF5F5F774B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:38:42.673955596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:38:42.676688952Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1BF7289F133, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:38:43.567311567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:38:43.569925677Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1BFA7C88546, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:38:44.468409265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:38:44.471179838Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1BFDD7ED7C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-07-03T15:38:44.535815345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:38:49.259572692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:38:49.262614855Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C0FB1751B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-03T15:38:49.535844739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:38:50.831819234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:38:50.834786151Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C158CDCC60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-07-03T15:38:50.834803989Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:38:50.834824139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:38:50.839732274Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C1591B98A6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:38:53.955377713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:38:53.960544626Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C2131C4694, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T15:38:54.535561721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:38:55.541358343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:38:55.544108847Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C2717ED818, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:38:56.334153305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:38:56.336726768Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C2A0BE8309, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:38:58.11768568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:38:58.124617851Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C30B4FBA45, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T15:38:59.53592018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:39:00.02230834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:39:00.02724783Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C37CB5B132, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:39:00.027268668Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:39:00.027288646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:39:00.029697435Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C37CDC84C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:39:01.072260366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:39:01.076308648Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C3BB3EC370, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:39:01.50259358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:39:01.50743081Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C3D4F0D305, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:39:01.949730021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:39:01.952793977Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C3EF7C02D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:39:04.297772447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:39:04.30136864Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C47B790CA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:39:04.305347078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:39:04.309905197Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C47BFD1205, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:39:04.516132773Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:39:04.534845088Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:39:04.534873656Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:39:04.534945823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:39:04.684444648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:39:04.686884471Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C4927585D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:39:06.171006664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:39:06.174122432Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C4EB160AD2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:39:07.669634962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:39:07.672402778Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C54465FB26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:39:08.207067202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:39:08.209856196Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C5646F4E12, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:39:08.209874954Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:39:08.209894985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:39:08.211995736Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C56492B64E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:39:08.455582873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:39:08.458148371Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C5733D01EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:39:09.535215956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:39:10.190912403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:39:10.194111273Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C5DAB37E28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:39:12.611870903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:39:12.617145167Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C66B208333, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-07-03T15:39:14.535208192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:39:14.543350622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:39:14.546016189Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C6DE197FCB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:39:15.150831943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:39:15.153713794Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C70251DEB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:39:18.29844712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:39:18.301169245Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C7BDEC71B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:39:18.584089759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:39:18.586892608Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C7CEF2F8C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:39:19.535014174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:39:20.524344082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:39:20.526962632Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C842980707, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-03T15:39:24.535353211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:39:25.163801325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:39:25.166723913Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C95724B094, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:39:27.627001344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:39:27.629549495Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1C9E9F0CE85, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:39:28.315499883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:39:28.318141246Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1CA12FB7B78, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:39:28.869607145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:39:28.872269746Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1CA3402D464, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T15:39:29.535738081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:39:34.535410056Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:39:34.535441165Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:39:34.535456844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:39:35.331170262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:39:35.333906042Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1CBB5281E7C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:39:37.44059384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:39:37.443189377Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1CC32E0EDCC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-07-03T15:39:39.535991948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:39:41.05034222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:39:41.052892439Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1CD0A09504D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:39:43.18151227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:39:43.184235641Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1CD8911B4EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:39:43.221425265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:39:43.223780171Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1CD8B6F4D2A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:39:43.264813086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:39:43.26706422Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1CD8E0354AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:39:43.267076952Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:39:43.267093372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:39:43.269024599Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1CD8E22E82D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=warn ts=2026-07-03T15:39:44.535676616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:39:45.343018796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:39:45.350177839Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1CE0A2C4655, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:39:45.938069997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:39:45.943730018Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1CE2D8DF85A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:39:46.081938593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:39:46.084509404Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1CE35F1CC03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:39:47.688985081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:39:47.691673781Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1CE95BC7F49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:39:48.36595095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:39:48.368935146Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1CEBE19B44E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:39:48.739127748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:39:48.742146997Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1CED458A8F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:39:49.083876656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:39:49.086588844Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1CEE8E0A1B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:39:49.18994049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:39:49.192386184Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1CEEF30E151, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:39:49.195432038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:39:49.197892647Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1CEEF860EAF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-03T15:39:49.535322191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:39:51.771331385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:39:51.776742304Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1CF89389B8D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:39:51.776765044Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:39:51.776789748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:39:51.779251497Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1CF8960AD3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-03T15:39:54.53493448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:39:59.535485966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:40:00.247248802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:40:00.251694667Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D1825D4D35, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:40:00.251710644Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:40:00.251732246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:40:00.257771468Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D182BCCBDD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:40:01.164782931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:40:01.169378142Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D1B910F83F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:40:02.567111579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:40:02.570082357Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D20C8C7880, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:40:03.799780937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:40:03.803335651Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D256106E98, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:40:04.515928011Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:40:04.535045209Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:40:04.535067829Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:40:04.535076496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:40:05.601648371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:40:05.606925071Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D2C190ED86, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:40:09.11605472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:40:09.118951772Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D392E5D954, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:40:09.11898378Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:40:09.119003899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:40:09.121226749Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D3930B6337, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-03T15:40:09.535643503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:40:09.609962595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:40:09.612593691Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D3B053CA5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:40:10.131307707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:40:10.134116331Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D3CF682402, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:40:10.495923407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:40:10.499050479Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D3E5275CA9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:40:10.918660456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:40:10.921375368Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D3FE555475, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:40:14.53587925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:40:15.216506742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:40:15.219376091Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D4FE831266, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:40:15.499923912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:40:15.503936771Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D50F7AFAAA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:40:16.354488341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:40:16.360600491Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D54289CD41, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-03T15:40:19.535389703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:40:21.534343552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:40:21.536906529Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D67712824D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:40:21.625161191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:40:21.627732831Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D67C7CB12C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:40:22.069931059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:40:22.072694204Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D697017DB6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:40:22.20617909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:40:22.208876236Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D69F201137, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:40:22.942711259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:40:22.945342566Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D6CB04B197, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=warn ts=2026-07-03T15:40:24.535564369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:40:28.64783524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:40:28.650538022Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D81F137C01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:40:29.318384371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:40:29.321453991Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D847106261, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-03T15:40:29.535651358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:40:30.220565621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:40:30.226714889Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D87D0627FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:40:32.692098958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:40:32.694919423Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D91024853B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:40:33.74990116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.6 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:40:33.757882401Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D94F7C02F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:40:34.535047931Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:40:34.535078719Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:40:34.53518861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:40:35.371690638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:40:35.377088239Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1D9B0008E61, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:40:37.254200466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:40:37.256891349Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1DA200ECDB0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-03T15:40:39.535531414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:40:39.881077096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:40:39.883618314Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1DABC9EEFE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:40:39.938810314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:40:39.941676779Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1DAC0145755, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:40:40.685564132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:40:40.685583096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:40:40.688187292Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1DAEC938E45, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:40:40.689363628Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1DAECA7D368, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:40:41.480429296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:40:41.483209289Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1DB1BF605EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:40:44.188335933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:40:44.192501379Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1DBBD735269, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:40:44.535498342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:40:46.376930514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:40:46.383129965Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1DC40046145, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:40:48.35126588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:40:48.354082559Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1DCB57F608B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-03T15:40:49.535066958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:40:51.628121675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:40:51.631731516Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1DD78DC4596, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-07-03T15:40:54.53583039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:40:56.475920039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:40:56.480689721Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1DE99E29A6A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:40:59.131167205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:40:59.133736293Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1DF380583E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:40:59.133754457Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:40:59.133772111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:40:59.135751645Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1DF3826AEC1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:40:59.535647599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-07-03T15:40:59.53566212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-03T15:40:59.538420305Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1DF502318B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:40:59.969250871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:40:59.971768251Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1DF69F860F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:41:02.13261425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:41:02.135569759Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1DFEAEFFA5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:41:04.515462638Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-03T15:41:04.535042119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:41:04.535047781Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:41:04.535073576Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:41:05.298907612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:41:05.301643044Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E0A7A726C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:41:06.640286908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="25 kB" avg_uncomp="13 kB" idle=1 full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:41:06.643008265Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E0F79A08D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:41:06.754301838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:41:06.756764691Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E0FE639527, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T15:41:09.535761672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:41:11.172000787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:41:11.175165455Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E205BCD06D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:41:13.085539696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:41:13.091559439Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E277F7A95D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:41:13.282319136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:41:13.285049818Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E2837F4365, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-07-03T15:41:13.285068458Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:41:13.285087161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:41:13.28714806Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E283A2A6AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:41:13.6363364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:41:13.639066162Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E29899B08D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:41:13.713383261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:41:13.715881528Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E29D2F3C3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:41:14.375532908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:41:14.378210204Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E2C4A7FE13, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:41:14.534910147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:41:15.907005018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:41:15.909918369Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E31FF36298, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:41:18.016012601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:41:18.018704671Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E39DA6B16D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-03T15:41:19.535737485Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:41:24.535385937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:41:25.974065062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:41:25.976613398Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E577FA290F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:41:29.005779826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:41:29.008877397Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E62CB51078, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-03T15:41:29.535439748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:41:29.768083042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:41:29.770707683Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E65A1F9883, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:41:29.770726343Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:41:29.7707448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:41:29.772837971Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E65A41D0BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:41:31.949111546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:41:31.9518991Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E6DC210362, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:41:31.973946724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.6 MB" total_uncomp="133 MB" avg_uncomp="33 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:41:31.982013014Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E6DDEB79B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:41:32.794175858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:41:32.798711498Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E70E9CD76C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:41:34.085578088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:41:34.088580771Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E75B7CBE42, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:41:34.535794087Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:41:34.535823298Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:41:34.535832788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:41:34.537604681Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-07-03T15:41:34.539670429Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.004712076s
level=info ts=2026-07-03T15:41:34.908991786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:41:34.913528778Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E78CA8C7FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:41:37.348293392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:41:37.353145801Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E81E11FF12, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:41:37.547787656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:41:37.550429526Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E829D49595, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-03T15:41:39.535153327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:41:40.581859799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:41:40.585030945Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E8DEB253AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:41:40.585048748Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:41:40.585067622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:41:40.587491624Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E8DEDB91A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:41:43.763491134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:41:43.766251962Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E99C51E6F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:41:43.766268189Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:41:43.766286267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:41:43.768381562Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E99C75A4A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:41:44.149424874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:41:44.152384923Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1E9B353F516, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-07-03T15:41:44.53526702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:41:45.844355813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:41:45.847647828Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1EA18601A1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:41:46.291818941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:41:46.294615276Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1EA3305A2B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:41:46.294634075Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:41:46.294654227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:41:46.296793729Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1EA3329F6CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:41:46.816584283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:41:46.820404605Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1EA525CA1AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:41:47.029904338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:41:47.034907835Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1EA5F25D84B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:41:47.821087209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:41:47.824090681Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1EA8E2E31C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-03T15:41:49.535206364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:41:54.193256576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="25 kB" avg_uncomp="13 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:41:54.196148642Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1EC09FC54A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:41:54.534883876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:41:57.437913941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:41:57.440794175Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1ECCB643FBD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:41:57.602323369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:41:57.604984567Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1ECD52D0353, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:41:58.424081806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:41:58.426938511Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1ED0629BAD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:41:58.426961871Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:41:58.427010239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:41:58.42969082Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1ED0651EACC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:41:58.623353514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:41:58.626067026Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1ED12080D60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:41:59.535680433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:42:00.023635571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:42:00.026394439Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1ED65810515, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:42:00.400554908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:42:00.403200173Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1ED7BF65EE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:42:00.403221127Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:42:00.403242815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:42:00.405336176Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1ED7C192AF3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:42:01.238618211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:42:01.241374583Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1EDADEA3BD4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:42:01.568608294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:42:01.571651704Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1EDC19B47E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:42:02.831827173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:42:02.837693593Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1EE0D11CA93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:42:03.133496863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:42:03.136230289Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1EE1EDC5484, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:42:04.515402104Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:42:04.534672857Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-03T15:42:04.534823474Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=info ts=2026-07-03T15:42:04.534833911Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-03T15:42:04.534852374Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.812µs success=true
level=warn ts=2026-07-03T15:42:04.534924499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:42:04.53494985Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:42:04.534986548Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:42:08.886097321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:42:08.888889913Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1EF75C032EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:42:09.535560087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:42:10.993793322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:42:10.996432878Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1EFF35DE3AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:42:11.036531368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:42:11.038877465Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1EFF5E73D29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:42:11.399423803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:42:11.402083088Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F00B8A902E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:42:12.166332279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:42:12.169122585Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F03943FF75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T15:42:14.535216845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:42:16.678877787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:42:16.681803145Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F1463CE266, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:42:17.706814684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:42:17.709575845Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F183809A26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:42:17.709591846Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:42:17.70961005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:42:17.711693496Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F183A362A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-03T15:42:19.535590784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:42:21.616495373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:42:21.619773839Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F26C8E53E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:42:21.66882906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:42:21.67153834Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F26FA80E3B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:42:21.671557347Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:42:21.67157629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:42:21.673916744Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F26FCEB063, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:42:22.032156717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.6 MB" total_uncomp="133 MB" avg_uncomp="33 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:42:22.039449179Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F2859512FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:42:22.306104838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:42:22.312918655Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F295E21CF5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:42:24.020553526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:42:24.023290525Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F2FBD38526, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:42:24.023309933Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:42:24.023332442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:42:24.025412597Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F2FBF723E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-03T15:42:24.535720756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:42:24.548814902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:42:24.552736163Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F31B633919, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:42:25.971695893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:42:25.976986466Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F370468B73, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:42:26.737147537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:42:26.739777084Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F39DBEF444, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:42:27.04084683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:42:27.043550497Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F3AFD9D9A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:42:29.153125388Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BED1F42D99CB37, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-03T15:42:29.535459815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:42:30.791539778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:42:30.794452353Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F48F6AA55A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:42:32.593741332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:42:32.596602609Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F4FAD6D85E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:42:34.535352948Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T15:42:34.535366542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:42:34.535390618Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:42:35.920710115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:42:35.923531024Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F5C121E3C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:42:37.553377424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:42:37.555921309Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F622700248, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:42:39.409068985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:42:39.411639794Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F6910C5D1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-03T15:42:39.535163055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:42:41.574606715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:42:41.577602404Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F71225A2F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:42:42.51301549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:42:42.516493358Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F74A1CBDC8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:42:42.57261915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:42:42.578636396Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F74DCFD6B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:42:42.673855122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:42:42.676611746Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F753A70C4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:42:42.927522043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:42:42.930315101Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F762C5F581, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-03T15:42:44.535763973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:42:45.745284582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:42:45.74790028Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F80AB7709F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:42:47.26549971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:42:47.268242946Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F86555567E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-03T15:42:49.535658039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:42:50.416350643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:42:50.419411532Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F92128DE36, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:42:50.724210263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="25 kB" avg_uncomp="13 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:42:50.729054647Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F9339D2FC6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:42:51.683159052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:42:51.685817325Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1F96CA47CBD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=warn ts=2026-07-03T15:42:54.535510113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:42:55.235615301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:42:55.238291741Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1FA40633F0C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:42:56.045679742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:42:56.048419315Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1FA70AD00C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:42:56.048436618Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:42:56.048453378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:42:56.050485683Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1FA70CF4BC6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:42:58.62802062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:42:58.630763609Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1FB0A97F512, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T15:42:59.535202875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:43:02.950889827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:43:02.953565493Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1FC0C4099B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:43:03.988732106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:43:03.991304662Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1FC4A1B1FD3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:43:03.99132365Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:43:03.991342126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:43:03.993440535Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1FC4A3E8D79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:43:04.515411718Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-03T15:43:04.535820587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:43:04.535824331Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:43:04.53584898Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:43:06.379918498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:43:06.382655342Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1FCD8A4319D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:43:06.972762004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:43:06.975413012Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1FCFBF9A210, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:43:07.395338549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:43:07.398213722Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1FD152B75B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:43:07.398231055Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:43:07.398249615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:43:07.400550404Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1FD15527FD4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:43:08.125486556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:43:08.128280826Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1FD40AF17CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T15:43:09.535392561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:43:10.23622918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:43:10.238935589Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1FDBE7EBCED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:43:11.012507177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:43:11.015237856Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1FDECC40143, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:43:12.539844257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:43:12.544353949Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1FE47E64333, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:43:14.535008441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:43:15.436661642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:43:15.442961455Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1FEF4AE4AAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:43:16.00610435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:43:16.009215246Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1FF166B3EFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:43:17.412448232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:43:17.41502303Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1FF6A396DFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:43:18.443128618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:43:18.446067549Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1FFA7ACA8AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:43:19.330494182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.6 MB" total_uncomp="133 MB" avg_uncomp="33 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:43:19.422466534Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED1FFE1DF9015, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T15:43:19.535310177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:43:20.028884924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:43:20.033455184Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED200064B72CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:43:20.033473079Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:43:20.03349199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:43:20.035597796Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED200066E7453, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:43:24.076630941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:43:24.079609596Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED200F7760EC4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-07-03T15:43:24.535388672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:43:26.572551417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:43:26.57620025Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2018C446C87, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:43:27.432543937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:43:27.437456325Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED201BF9ABED9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:43:29.393770854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:43:29.396875068Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED202346397F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-07-03T15:43:29.535286415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:43:32.79559012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:43:32.79825298Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED202FF229D3C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:43:32.898543938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:43:32.901070778Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2030543FA1D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:43:33.016575555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:43:33.019053584Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2030C4C1C7A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-07-03T15:43:34.535526586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:43:34.535553576Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:43:34.535587086Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:43:35.65869646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:43:35.661419722Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED203A9CAFCEB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:43:35.661440661Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:43:35.661460833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:43:35.663498368Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED203A9EDBA58, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:43:36.346684592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:43:36.349361011Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED203D2CC0D8A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:43:37.173527388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:43:37.17629437Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20404169B21, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:43:38.189341646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:43:38.191955853Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20440A1193B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:43:38.197999453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:43:38.200162333Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED204411F5E0A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:43:39.535390059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:43:40.984487621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:43:40.987194842Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED204E73B4059, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:43:43.339422892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:43:43.342397529Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED205739C8E02, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-03T15:43:44.535805405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:43:46.013126564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:43:46.015774124Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20612F6E003, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:43:46.015789682Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:43:46.015807148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:43:46.017811349Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20613181AAF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:43:47.25016536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:43:47.252806503Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2065CB17748, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:43:48.378215294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:43:48.38118167Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2069FF2F9C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:43:48.4243264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="25 kB" avg_uncomp="13 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:43:48.430283938Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED206A2E12527, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:43:49.535275322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:43:51.410479465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:43:51.413012507Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20754A9768D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:43:52.712015326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:43:52.714709177Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED207A23FD058, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:43:52.745747787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:43:52.748083222Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED207A43E5CD2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:43:52.943629548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:43:52.946046182Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED207B009C3DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:43:54.535801272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:43:57.967830618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:43:57.970518674Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED208DB84D05D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-03T15:43:59.535513222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:44:00.329006999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:44:00.331709808Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2096842036F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:44:04.51629469Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-03T15:44:04.535643971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:44:04.535669162Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:44:04.535700364Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:44:04.567953213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:44:04.571960252Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20A64FEEDD5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:44:06.59919494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:44:06.603990473Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20ADE1C85A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:44:06.663073289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:44:06.665664511Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20AE1CA4697, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:44:07.667141924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:44:07.669865808Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20B1DA46504, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-03T15:44:09.535347558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:44:12.205364373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:44:12.208254745Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20C2C276135, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:44:13.63259483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:44:13.6356102Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20C8138D359, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-03T15:44:14.534986797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:44:14.53721621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.6 MB" total_uncomp="133 MB" avg_uncomp="33 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:44:14.544544456Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20CB7667789, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:44:14.690800099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:44:14.695315563Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20CC065033A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:44:19.131708651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:44:19.134565105Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20DC8FD739D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-07-03T15:44:19.535914678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:44:19.613218608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:44:19.615771561Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20DE5AD8CD8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:44:19.821645086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:44:19.824700468Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20DF21F9A3A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:44:21.7616917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:44:21.764400417Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20E65BE21C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:44:21.834724608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:44:21.837583682Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20E6A198744, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:44:21.928946603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:44:21.931759472Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20E6FB8E284, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:44:22.675866704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:44:22.679337752Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20E9C480308, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:44:22.960093456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:44:22.965016721Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20EAD4E153D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=warn ts=2026-07-03T15:44:24.535654794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:44:27.210485725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:44:27.213275604Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20FAA85838B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:44:27.281340408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:44:27.283832407Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20FAEBB680A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:44:27.410302865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:44:27.412820789Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED20FB66B38AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:44:28.975918663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:44:28.97880776Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21013C1D9E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:44:29.518009744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="25 kB" avg_uncomp="13 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:44:29.520753331Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED210340D3091, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:44:29.535791335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:44:31.797210098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:44:31.8000821Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED210BBE8F7A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:44:32.445807584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:44:32.448583666Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED210E2921895, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:44:34.534841847Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:44:34.534872357Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:44:34.534877383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:44:38.847489986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:44:38.850164026Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21260226CE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:44:39.423734138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:44:39.42665517Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED212827FE18E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-03T15:44:39.53536382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:44:44.535634147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:44:44.742070551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:44:44.748475359Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED213BFB31B51, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:44:47.958304495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:44:47.96101055Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2147F2E3E92, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-03T15:44:49.535043352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:44:53.667724109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:44:53.672393628Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED215D39886A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:44:54.534895301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:44:55.924487095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:44:55.929275042Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2165A1FF4DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:44:57.00101014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:44:57.003790753Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2169A2CEE16, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:44:59.346143782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:44:59.348846531Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21725F3A514, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-07-03T15:44:59.535849817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:45:00.157382483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:45:00.160342749Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21756500537, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:45:00.84144831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:45:00.844071389Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2177F123509, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:45:02.609502284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.6 MB" total_uncomp="133 MB" avg_uncomp="33 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:45:02.616768474Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED217E8BBC2FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:45:02.616791617Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:45:02.616817716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.6 MB" total_uncomp="133 MB" avg_uncomp="33 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:45:02.626016025Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED217E9484E0D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:45:03.333555204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:45:03.338410822Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21813BED5C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:45:03.338428657Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:45:03.338449152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:45:03.340585042Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21813E28692, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:45:04.515435539Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:45:04.535435523Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:45:04.535460813Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:45:04.535499151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:45:05.055637075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:45:05.058856961Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2187A49BD79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:45:05.818220432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:45:05.821784004Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED218A7C2F894, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:45:05.821803887Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:45:05.821829058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:45:05.826586371Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED218A810758E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:45:05.874653094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:45:05.878669114Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED218AB291174, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:45:07.580573938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:45:07.583096143Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21910C13883, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-03T15:45:07.706414896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:45:07.708838418Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED219183F984A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-03T15:45:09.535467782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:45:09.802201574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="25 kB" avg_uncomp="13 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:45:09.804939381Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED219952E4238, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:45:09.804960584Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:45:09.804997056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="25 kB" avg_uncomp="13 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:45:09.807179489Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2199551475B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:45:11.303426905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:45:11.305950011Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED219EEA6A340, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:45:11.642018529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=info ts=2026-07-03T15:45:11.643212871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:45:11.644727702Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21A02D7B067, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-07-03T15:45:11.645699339Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21A02E77CBF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:45:12.669450856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:45:12.672280209Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21A401659BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:45:13.153695153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:45:13.156260517Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21A5CEFBE13, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:45:13.207412026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:45:13.209759756Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21A6021C79B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:45:14.027091631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:45:14.029657047Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21A90FED7CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:45:14.343517715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:45:14.3460137Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21AA3DA75F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-07-03T15:45:14.535165885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:45:17.122244668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.6 MB" total_uncomp="133 MB" avg_uncomp="33 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:45:17.129213864Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21B49BDB7DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T15:45:19.535600736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:45:19.557748243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:45:19.560515264Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21BDAA7A2FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:45:19.920735257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="25 kB" avg_uncomp="13 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:45:19.923365158Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21BF048C32E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:45:20.245863601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:45:20.249429577Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21C03B88F9D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:45:21.927726821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:45:21.933239082Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21C6815C57B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:45:24.523337781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:45:24.52641032Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21D02A5AB53, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-07-03T15:45:24.526426555Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:45:24.52644604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:45:24.530757623Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21D02EAF0EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-03T15:45:24.535877547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:45:27.130517728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:45:27.135615612Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21D9E2C4AEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:45:29.535356146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:45:31.2849127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:45:31.287556487Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED21E95A59D01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:45:34.535708672Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:45:34.535737066Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:45:34.535749577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:45:37.465002343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:45:37.469150852Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2200617BAB0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:45:39.5359748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:45:41.955306087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:45:41.961891075Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22111E2A258, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:45:42.995181841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.6 MB" total_uncomp="133 MB" avg_uncomp="33 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:45:43.002019543Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2214FE1D4C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:45:44.034081964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:45:44.039476922Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2218DB6F066, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:45:44.117724076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="25 kB" avg_uncomp="13 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:45:44.121989657Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22192A31032, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:45:44.535827013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:45:45.566339339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:45:45.568821294Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED221E8E10122, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:45:47.419072378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:45:47.421940212Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22257540BFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:45:48.100624046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:45:48.103281379Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2227FF08269, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:45:49.074764309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:45:49.077771762Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED222BA05DCD6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:45:49.077789442Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:45:49.077807938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:45:49.080237302Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED222BA2C7A45, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-07-03T15:45:49.535954103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:45:50.83172382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:45:50.834540888Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22322BBC1B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:45:51.233080452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:45:51.235666927Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2233AA63581, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:45:52.511609713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:45:52.514343654Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22386DBF6CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:45:52.660726479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:45:52.663553534Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2238FC1397D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:45:54.535038561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:45:58.783730488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:45:58.787291558Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED224FCC1D593, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-07-03T15:45:59.535643381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:46:00.46624469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:46:00.472499407Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2256133B1B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:46:00.625698914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:46:00.628276118Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2256A7CEBFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:46:00.918265973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:46:00.921077255Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2257BF0D276, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:46:02.877976562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:46:02.880682154Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED225F0BD5D0A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:46:02.88069848Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:46:02.880716725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:46:02.882829808Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED225F0E1393C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:46:02.88889915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:46:02.891508755Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED225F1629D63, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:46:03.146474887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:46:03.149678571Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22600C3F50F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:46:03.194725402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:46:03.197517177Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22603A04FF3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:46:03.726131979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:46:03.728821795Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED226234B59EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-03T15:46:04.318336773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:46:04.320942708Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED226469734F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:46:04.51606139Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:46:04.535444007Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:46:04.535467262Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-03T15:46:04.535483691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:46:04.931080137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:46:04.933842065Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2266B1C496E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-03T15:46:09.535154619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:46:09.705672942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:46:09.70838752Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22787B46DFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:46:11.02023303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:46:11.022815577Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED227D60DCA71, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-03T15:46:12.975777256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:46:12.978823368Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2284AA3BE25, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-03T15:46:14.535082273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-03T15:46:18.920142559Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BED229ACC78980, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-03T15:46:19.535873004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:46:20.701414968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:46:20.706150307Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22A1736D01C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:46:21.109932698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:46:21.11279343Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22A2F75F8C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-03T15:46:23.080510585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:46:23.084670493Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22AA4FEA4EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:46:23.08469014Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:46:23.084713859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:46:23.090244459Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22AA5560BD3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-03T15:46:24.535100599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:46:26.496010754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:46:26.500847648Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22B709CC9C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-03T15:46:27.954913703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:46:27.957637526Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22BC771CC05, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-03T15:46:29.535348558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:46:31.480184855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:46:31.483064785Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22C9992FC84, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:46:34.534997335Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T15:46:34.535001334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:46:34.535027136Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:46:34.541582306Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-07-03T15:46:34.543675257Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008665824s
level=info ts=2026-07-03T15:46:34.634669293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.6 MB" total_uncomp="133 MB" avg_uncomp="33 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:46:34.64168054Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22D55D8DBCF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-03T15:46:35.879012711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:46:35.883725975Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22D9FE05D26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:46:37.462322668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:46:37.464834534Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22DFE1EF6A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:46:38.014696164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:46:38.018701039Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22E1F226245, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:46:38.636340937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:46:38.641481737Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22E443DFC10, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-07-03T15:46:39.535786603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:46:40.341776546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:46:40.344452784Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22EA9C18AE7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:46:41.004211816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="25 kB" avg_uncomp="13 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:46:41.006827599Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22ED13CA489, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-03T15:46:41.879196818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:46:41.88273364Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22F0571CE3C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:46:42.547005994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:46:42.551806831Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22F2D536B85, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:46:42.582847907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:46:42.585668101Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22F2F571DD8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:46:43.234602177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:46:43.237515635Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22F56306AE1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:46:44.156835145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:46:44.159503545Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22F8D26E433, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:46:44.535223277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:46:45.425415546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:46:45.428626245Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED22FD8C2B825, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:46:47.603619806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:46:47.606415079Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2305A991439, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:46:47.606432272Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:46:47.606454155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:46:47.608778985Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2305ABFFACA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-03T15:46:49.535473636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:46:50.512682874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:46:50.515212161Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23107FB6D32, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:46:52.85289258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:46:52.855604752Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED231937A7301, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:46:54.497080129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:46:54.499617117Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED231F57943EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-03T15:46:54.535828312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:46:54.802916402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:46:54.805474721Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23207B3A6CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:46:54.945318641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:46:54.947951607Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2321030CA86, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:46:55.58269089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:46:55.585607079Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED232363261A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:46:55.585626047Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:46:55.585645106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:46:55.58814229Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2323659D67F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:46:57.268504425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:46:57.271182375Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2329AAA60F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:46:58.11056526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:46:58.113109629Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED232CCD9A3E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T15:46:59.535607051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:47:04.51541844Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-03T15:47:04.535480386Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-03T15:47:04.535516379Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-03T15:47:04.535525962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:47:04.535543495Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:47:04.535588613Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=info ts=2026-07-03T15:47:04.535596589Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-03T15:47:04.535613679Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.495µs success=true
level=info ts=2026-07-03T15:47:05.555785597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="297 kB" avg_comp="297 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-03T15:47:05.558870619Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23488A6B499, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-03T15:47:05.973194663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="660 kB" total_uncomp="7.2 MB" avg_uncomp="3.6 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-03T15:47:05.979527528Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED234A1BA2BF8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-03T15:47:07.405661352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:47:07.410627507Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED234F70732AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:47:08.651717699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-03T15:47:08.654493551Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED235412B5ADF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-03T15:47:09.535772093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:47:12.191605139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:47:12.1947947Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED236142EE7F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:47:14.137393002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="230 kB" avg_uncomp="230 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-03T15:47:14.140166243Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23688238B2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-03T15:47:14.535159294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:47:19.53588591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:47:20.960000771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="162 kB" avg_uncomp="162 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-03T15:47:20.962648552Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED2381ECB11B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-03T15:47:23.181329047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="97 kB" avg_comp="97 kB" total_uncomp="641 kB" avg_uncomp="641 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-03T15:47:23.184180495Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED238A333D624, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-03T15:47:23.683701173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.6 MB" total_uncomp="133 MB" avg_uncomp="33 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-03T15:47:23.691009586Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED238C169CA98, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-03T15:47:24.535360521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:47:26.768513241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="614 kB" avg_comp="614 kB" total_uncomp="4.9 MB" avg_uncomp="4.9 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-03T15:47:26.775799149Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23979475493, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-03T15:47:28.773265374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-03T15:47:28.778012313Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED239F09FBFAA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-03T15:47:29.350449727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="25 kB" avg_uncomp="13 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-03T15:47:29.353163924Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23A12E6552B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-03T15:47:29.53535712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:47:29.914834538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-03T15:47:29.917291527Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23A3488B1BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-03T15:47:29.962333042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=91a74a3bdb9936ff immediate=false num_chunks=1 total_comp="15 kB" avg_comp="15 kB" total_uncomp="130 kB" avg_uncomp="130 kB" full=1 labels="{stream=\"openshift-console\"}"
level=error ts=2026-07-03T15:47:29.964740552Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23A375CEF04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console\"}"
level=info ts=2026-07-03T15:47:31.602946852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="915 kB" avg_uncomp="915 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-03T15:47:31.605805114Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23A992BFBBC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-03T15:47:31.792106327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="22 kB" avg_comp="22 kB" total_uncomp="207 kB" avg_uncomp="207 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-07-03T15:47:31.794813193Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23AA46FC0A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-07-03T15:47:32.43392863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="160 kB" avg_comp="160 kB" total_uncomp="786 kB" avg_uncomp="786 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-03T15:47:32.436894279Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23ACAB4CB79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-03T15:47:32.836104567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-03T15:47:32.83871692Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23AE2A8E8CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-03T15:47:34.535786787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:47:34.535784449Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-03T15:47:34.535814827Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-03T15:47:34.617358617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="582 kB" avg_uncomp="582 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-03T15:47:34.620132063Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23B4CD61223, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-03T15:47:35.433410635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="224 kB" avg_comp="224 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-03T15:47:35.436390857Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23B7D7D3989, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-03T15:47:35.671430743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-03T15:47:35.674144666Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23B8BA85954, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-03T15:47:35.762203151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="1.3 MB" avg_uncomp="1.3 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-07-03T15:47:35.764933628Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23B9112B236, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-07-03T15:47:38.117735133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:47:38.120418175Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23C1D794A54, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:47:38.120441359Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:47:38.120461382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:47:38.122582976Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23C1D9C848D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:47:38.862670204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:47:38.865252731Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23C49DEA57F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:47:38.865269253Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:47:38.865287548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:47:38.867410571Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23C4A007643, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-03T15:47:39.535036164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-03T15:47:44.535926731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:47:44.668240288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-03T15:47:44.670981874Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23DA3E95C67, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-03T15:47:44.764232558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="81 kB" avg_uncomp="81 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-03T15:47:44.76681326Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23DA99FFE6B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-03T15:47:48.561639908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-03T15:47:48.564391649Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23E8BFBF807, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-03T15:47:49.534893895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-03T15:47:49.735266282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-07-03T15:47:49.73772771Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23ED1EB7741, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-07-03T15:47:49.869302253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:47:49.871774502Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23ED9E9970C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-07-03T15:47:49.871788578Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-03T15:47:49.871805638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-03T15:47:49.873792694Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23EDA0A2CCA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-03T15:47:53.073763059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-07-03T15:47:53.076648907Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23F98EE4E4A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-07-03T15:47:53.134987983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-03T15:47:53.137504248Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BED23F9C906DCF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-03T15:47:54.535818243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"