level=info ts=2026-02-16T18:09:13.045404444Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-02-16T18:09:13.045439601Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-02-16T18:09:13.045727087Z caller=modules.go:1984 msg="initializing ingester query tags interceptors"
level=info ts=2026-02-16T18:09:13.04654502Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-02-16T18:09:13.046926331Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T18:09:13.046999149Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.53 inf=eth0
level=error ts=2026-02-16T18:09:13.052065608Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T18:09:13.052087804Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-02-16T18:09:13.054185852Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-02-16T18:09:13.054257283Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:09:13.054531425Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-02-16T18:09:13.05454314Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-02-16T18:09:13.05454231Z
level=debug ts=2026-02-16T18:09:13.054581163Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-02-16T18:09:13.054593343Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-02-16T18:09:13.054603976Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-02-16T18:09:13.054608003Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-02-16T18:09:13.054607515Z
level=debug ts=2026-02-16T18:09:13.054611815Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-02-16T18:09:13.05531793Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-d7290ce8
ts=2026-02-16T18:09:13.055370617Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-02-16T18:09:13.055387417Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.53 advertisePort=7946
level=debug ts=2026-02-16T18:09:13.05541364Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.53 advertisePort=7946
level=info ts=2026-02-16T18:09:13.05556367Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=debug ts=2026-02-16T18:09:13.056073612Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T18:09:13.056136107Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.128.0.53 inf=eth0
ts=2026-02-16T18:09:13.060448244Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T18:09:13.060463047Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T18:09:13.060473351Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=4.912308ms
level=error ts=2026-02-16T18:09:13.060480258Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-02-16T18:09:13.060491892Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-02-16T18:09:13.062637366Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:09:13.062647511Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T18:09:13.069596704Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=debug ts=2026-02-16T18:09:13.069657751Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-02-16T18:09:13.06965199Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-02-16T18:09:13.069672216Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-02-16T18:09:13.069692572Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-02-16T18:09:13.069698102Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=info ts=2026-02-16T18:09:13.069718499Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-02-16T18:09:13.069729683Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-02-16T18:09:13.069745119Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-02-16T18:09:13.06975115Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-02-16T18:09:13.070226103Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-02-16T18:09:13.070236019Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-02-16T18:09:13.070254674Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-02-16T18:09:13.070285893Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-02-16T18:09:13.070338099Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-02-16T18:09:13.070379998Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-02-16T18:09:13.070384482Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=debug ts=2026-02-16T18:09:13.070388194Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=info ts=2026-02-16T18:09:13.070391141Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-02-16T18:09:13.070496029Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-02-16T18:09:13.070510632Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-02-16T18:09:13.070516386Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-02-16T18:09:13.070521499Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-02-16T18:09:13.070526325Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-02-16T18:09:13.070530687Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-02-16T18:09:13.070548246Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint"
level=info ts=2026-02-16T18:09:13.070588303Z caller=recovery.go:42 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-02-16T18:09:13.070658092Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=112.86µs errors=false
level=info ts=2026-02-16T18:09:13.070666929Z caller=ingester.go:587 component=ingester msg="recovering from WAL"
level=info ts=2026-02-16T18:09:13.070769716Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=224.555µs errors=false
level=info ts=2026-02-16T18:09:13.070776242Z caller=ingester.go:551 component=ingester msg="closing recoverer"
level=info ts=2026-02-16T18:09:13.070784538Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=239.419µs
level=info ts=2026-02-16T18:09:13.070814708Z caller=wal.go:158 msg=started component=wal
level=info ts=2026-02-16T18:09:13.07082071Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-02-16T18:09:13.070840546Z caller=lifecycler.go:714 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-02-16T18:09:13.07083455Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=21.956880958s
level=info ts=2026-02-16T18:09:13.070906366Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 1m0s with no heartbeat"
level=info ts=2026-02-16T18:09:13.070917199Z caller=loki.go:599 msg="Loki started" startup_time=84.699581ms
level=debug ts=2026-02-16T18:09:13.07091979Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-02-16T18:09:13.070934114Z caller=lifecycler.go:556 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=debug ts=2026-02-16T18:09:13.070992839Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-02-16T18:09:14.091801013Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:09:14.091828243Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T18:09:17.783535067Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:09:17.78355883Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T18:09:18.071658141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:09:22.175972033Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:09:22.175996918Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T18:09:23.071750173Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:09:27.427677426Z caller=logging.go:137 msg="GET /ready (503) 124.905µs"
level=warn ts=2026-02-16T18:09:28.071404638Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:09:33.07154993Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:09:37.425807921Z caller=logging.go:137 msg="GET /ready (503) 74.398µs"
ts=2026-02-16T18:09:37.588996633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.76:7946"
ts=2026-02-16T18:09:37.591810134Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused"
ts=2026-02-16T18:09:37.593311265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.72:7946"
ts=2026-02-16T18:09:37.596260187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.75:7946"
ts=2026-02-16T18:09:37.598832844Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.74:7946: dial tcp 10.129.0.74:7946: connect: connection refused"
ts=2026-02-16T18:09:37.600971635Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.58:7946: dial tcp 10.130.0.58:7946: connect: connection refused"
level=info ts=2026-02-16T18:09:37.600982436Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=24.540486242s
level=warn ts=2026-02-16T18:09:38.071817688Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:09:43.070916779Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-02-16T18:09:43.070961172Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-02-16T18:09:43.070973586Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:09:43.070996539Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:09:47.274511019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:60880"
level=warn ts=2026-02-16T18:09:48.071797065Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:09:50.452523831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:60714"
level=warn ts=2026-02-16T18:09:53.071196871Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:09:57.564638438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:54948"
level=warn ts=2026-02-16T18:09:58.071599927Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:10:03.071803133Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:10:06.351141439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33892"
level=warn ts=2026-02-16T18:10:08.071937915Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:10:10.668935304Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:10:11.084214402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=debug ts=2026-02-16T18:10:13.053583174Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:10:13.05470782Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:10:13.070980741Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:10:13.071013992Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:10:13.071025334Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:10:13.071033173Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:10:16.763950781Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T18:10:17.277678902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:54562"
level=warn ts=2026-02-16T18:10:18.07171041Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:10:23.071263873Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:10:28.071014084Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:10:33.071683646Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:10:36.354284399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46846"
level=debug ts=2026-02-16T18:10:37.699239414Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T18:10:38.071325822Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:10:41.087322963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:10:43.071467487Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:10:43.071496763Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:10:43.071505886Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:10:43.071524645Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:10:45.804938761Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:10:47.280283001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:59616"
level=warn ts=2026-02-16T18:10:48.072054653Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:10:53.07202013Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:10:58.071361348Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:10:59.907698139Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T18:11:03.071312018Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:11:08.071356897Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:11:11.090589866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=debug ts=2026-02-16T18:11:13.053435062Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:11:13.054497293Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:11:13.071888518Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:11:13.07191472Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:11:13.07192444Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:11:13.071942271Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:11:13.562121639Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:11:13.562122301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:11:13.571997376Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:11:18.071609983Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:11:22.84029712Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T18:11:23.071418857Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:11:27.975650884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:11:27.984123501Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:11:28.071604794Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:11:33.07103158Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:11:38.072001595Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:11:38.884462542Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:11:41.093355614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:11:43.07137924Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:11:43.071411975Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:11:43.071423211Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:11:43.071438308Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:11:48.071467582Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:11:52.346309158Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T18:11:52.348337416Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:11:53.071230922Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:11:58.071722261Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:11:58.63491026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:11:58.643266876Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:12:03.071072242Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:12:06.360189116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50334"
level=warn ts=2026-02-16T18:12:08.071257645Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:12:08.526990561Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:12:11.096375894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=debug ts=2026-02-16T18:12:13.05392695Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:12:13.054997717Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:12:13.071486605Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:12:13.071512839Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:12:13.071521517Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:12:13.071570446Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:12:18.071438253Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:12:19.367293944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:48762"
level=warn ts=2026-02-16T18:12:23.071947388Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:12:28.071434011Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:12:31.779173837Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T18:12:33.071330214Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:12:37.785968442Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:12:38.071283402Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:12:41.099746148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:12:43.071892741Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:12:43.071920249Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:12:43.071928954Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:12:43.071937743Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:12:47.287591816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:54006"
level=warn ts=2026-02-16T18:12:48.071201253Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:12:52.103533713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:12:52.111009623Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:12:53.071091191Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:12:58.071552576Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:13:03.071173126Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:13:04.035543343Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:13:06.364757572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:44618"
level=warn ts=2026-02-16T18:13:08.071501264Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:13:11.102208207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:13:13.05455348Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T18:13:13.055068732Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:13:13.07110711Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:13:13.071127782Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:13:13.071136739Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:13:13.071138195Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:13:18.071400586Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:13:19.37156565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:53808"
level=warn ts=2026-02-16T18:13:23.071800284Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:13:28.071537872Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:13:32.425033875Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:13:33.071785941Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:13:34.616567257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T18:13:37.51397345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:13:37.521620068Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:13:38.071483293Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:13:41.104809154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:13:43.071549581Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:13:43.071551502Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:13:43.07157754Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:13:43.071585961Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:13:48.071623221Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:13:49.374788978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:34318"
level=warn ts=2026-02-16T18:13:53.071725955Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:13:58.071641806Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:14:03.071589214Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:14:03.343597271Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:14:06.369810374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57156"
level=warn ts=2026-02-16T18:14:08.071634209Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:14:11.107356163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=debug ts=2026-02-16T18:14:13.0542387Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:14:13.05531472Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:14:13.071766183Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T18:14:13.071768426Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:14:13.07178922Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:14:13.071797471Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:14:13.071814036Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:14:13.071911946Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=debug ts=2026-02-16T18:14:13.074016824Z caller=checkpoint.go:378 msg="writing series" size="3.9 MB"
level=debug ts=2026-02-16T18:14:13.074028224Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.9 MB" series=1
ts=2026-02-16T18:14:17.293248287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:35668"
level=debug ts=2026-02-16T18:14:17.572763923Z caller=checkpoint.go:378 msg="writing series" size="173 kB"
level=warn ts=2026-02-16T18:14:18.071160294Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:14:19.351697191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:14:19.361972051Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:14:22.073051205Z caller=checkpoint.go:378 msg="writing series" size="92 kB"
level=warn ts=2026-02-16T18:14:23.071021975Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:14:26.572633409Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-02-16T18:14:28.071902219Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:14:29.047253369Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:14:31.072136895Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=warn ts=2026-02-16T18:14:33.071712275Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:14:35.573119885Z caller=checkpoint.go:378 msg="writing series" size="515 B"
ts=2026-02-16T18:14:36.373475127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50846"
level=warn ts=2026-02-16T18:14:38.071358883Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:14:40.072610861Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
ts=2026-02-16T18:14:41.110021427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T18:14:43.071474327Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:14:43.071473683Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:14:43.071498648Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:14:43.071508615Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:14:44.061275773Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T18:14:44.572351957Z caller=checkpoint.go:378 msg="writing series" size="298 kB"
level=warn ts=2026-02-16T18:14:48.071419524Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:14:49.073243353Z caller=checkpoint.go:378 msg="writing series" size="164 kB"
level=warn ts=2026-02-16T18:14:53.071628219Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:14:53.572625813Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=info ts=2026-02-16T18:14:55.834988876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="47 kB" avg_uncomp="47 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:14:55.83499945Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-02-16T18:14:55.842564485Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:14:56.135847283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="12 kB" avg_uncomp="12 kB" idle=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:14:56.139459557Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:14:56.435290138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="56 kB" avg_comp="56 kB" total_uncomp="56 kB" avg_uncomp="56 kB" idle=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:14:56.438977955Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c67a46ea4%3A19c67a4d66e%3Ab924d97a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T18:14:56.735824151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" idle=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:14:56.739848208Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:14:57.03568092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="8.1 kB" avg_comp="8.1 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" idle=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:14:57.039491213Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:14:57.33549433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:14:57.34047247Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:14:57.635646777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" idle=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:14:57.638981373Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:14:57.935817878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" idle=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:14:57.938890387Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T18:14:58.071369506Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:14:58.072490264Z caller=checkpoint.go:378 msg="writing series" size="127 kB"
level=debug ts=2026-02-16T18:15:02.572964817Z caller=checkpoint.go:378 msg="writing series" size="107 kB"
level=debug ts=2026-02-16T18:15:02.572994295Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-02-16T18:15:03.071325596Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:15:07.072984024Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=warn ts=2026-02-16T18:15:08.071646563Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:15:08.251170791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:15:08.259829926Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:15:09.894123462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:15:09.898274404Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c67a46ea4%3A19c67a4d66e%3Ab924d97a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
ts=2026-02-16T18:15:11.112862887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=debug ts=2026-02-16T18:15:11.572563127Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
level=info ts=2026-02-16T18:15:11.864628799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:15:11.868302005Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T18:15:13.053904426Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:15:13.054964566Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T18:15:13.055176583Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-02-16T18:15:13.055224291Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1968072 n=1
level=debug ts=2026-02-16T18:15:13.055267743Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1968072 n=1
level=info ts=2026-02-16T18:15:13.07116768Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:15:13.071185685Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:15:13.071193396Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:15:13.071218938Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:15:13.268971314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:15:13.272764309Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:15:14.114277491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:15:14.117443746Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:15:14.951437393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:15:14.954763223Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:15:15.48826483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:15:15.49290601Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:15:15.691493999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:15:15.695470151Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:15:15.985195278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:15:15.988413474Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:15:16.073035441Z caller=checkpoint.go:378 msg="writing series" size="424 kB"
ts=2026-02-16T18:15:17.297858741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:55968"
level=warn ts=2026-02-16T18:15:18.071650738Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:15:20.573032623Z caller=checkpoint.go:378 msg="writing series" size="7.9 kB"
level=warn ts=2026-02-16T18:15:23.071196355Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:15:24.774449266Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:15:24.774496686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="2.4 kB" avg_comp="2.4 kB" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:15:24.777884033Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:15:25.072935331Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=info ts=2026-02-16T18:15:25.075001505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="193 B" avg_comp="193 B" total_uncomp="274 B" avg_uncomp="274 B" idle=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:15:25.078909169Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:15:25.374711715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:15:25.378080148Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:15:25.675021577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="5.0 kB" avg_comp="5.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:15:25.678502739Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:15:25.975282402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:15:25.980238255Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:15:26.275182088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="3.1 kB" avg_comp="3.1 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:15:26.278181101Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:15:26.574943208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:15:26.578331909Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:15:28.071162066Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:15:29.572100095Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-02-16T18:15:33.071127224Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:15:34.072771387Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=debug ts=2026-02-16T18:15:34.975338008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T18:15:35.814498258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:15:35.81772167Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T18:15:36.378249762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:44802"
level=info ts=2026-02-16T18:15:37.138598431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:15:37.141995753Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:15:38.071510486Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:15:38.572367369Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=info ts=2026-02-16T18:15:40.584593315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:15:40.588133508Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-02-16T18:15:41.115742873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:15:41.387702559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:15:41.391135395Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:15:41.656971805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:15:41.660393391Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:15:42.033166738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:15:42.036433926Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:15:42.567095669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:15:42.570543322Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c67a46ea4%3A19c67a4d66e%3Ab924d97a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T18:15:42.995797511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:15:42.999021536Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:15:43.071356863Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:15:43.071358738Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:15:43.071378765Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:15:43.071387232Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:15:43.072458156Z caller=checkpoint.go:378 msg="writing series" size="1.6 kB"
level=info ts=2026-02-16T18:15:43.889142984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:15:43.892443093Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:15:46.401782901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:15:46.405251871Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T18:15:47.572735106Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=warn ts=2026-02-16T18:15:48.071752498Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:15:48.592634529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:15:48.596043387Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:15:48.847876669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:15:48.852602374Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:15:49.461391204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:15:49.464503479Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:15:49.626740905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:15:49.632640658Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:15:51.866860871Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:15:51.866916292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="232 B" avg_comp="232 B" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" idle=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:15:51.870447736Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:15:52.0731244Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T18:15:52.16740815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="487 B" avg_comp="487 B" total_uncomp="487 B" avg_uncomp="487 B" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:15:52.170548057Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:15:52.467512927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="10 kB" avg_uncomp="10 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T18:15:52.470597154Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c67a56bbf%3A19c67a599ae%3A3ce93348?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T18:15:52.691194805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:15:52.694397153Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T18:15:53.071651437Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:15:54.281409477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:15:54.284698454Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:15:56.572596567Z caller=checkpoint.go:378 msg="writing series" size="93 kB"
level=warn ts=2026-02-16T18:15:58.071574648Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:15:58.931048645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:15:58.93691096Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:16:01.073141474Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=warn ts=2026-02-16T18:16:03.071210986Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:16:03.094277803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:16:03.09818982Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:16:04.863496215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:16:04.866954161Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:16:05.004196923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:16:05.007989939Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:16:05.572571661Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T18:16:07.047715487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:16:07.051250469Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:16:08.070990851Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:16:09.092809934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T18:16:09.09649732Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c67a56bbf%3A19c67a599ae%3A3ce93348?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T18:16:09.120567994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:16:09.124478011Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:16:10.07258412Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
ts=2026-02-16T18:16:11.118570364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:16:12.647872674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:16:12.651317331Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:16:13.053704861Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:16:13.05474201Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:16:13.071160662Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:16:13.071174332Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:16:13.07118294Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:16:13.071204081Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:16:13.211471342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:16:13.21459483Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:16:14.572365933Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=warn ts=2026-02-16T18:16:18.071247405Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:16:18.471696727Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:16:19.07250765Z caller=checkpoint.go:378 msg="writing series" size="62 kB"
ts=2026-02-16T18:16:19.382632766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:39540"
level=info ts=2026-02-16T18:16:22.465300995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="917 B" avg_comp="917 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:16:22.469128621Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:16:23.071175952Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:16:23.572761227Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
level=debug ts=2026-02-16T18:16:23.572784219Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=18
level=warn ts=2026-02-16T18:16:28.071405802Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:16:28.072477703Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T18:16:31.206048202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:16:31.211697099Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T18:16:32.572516843Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-02-16T18:16:33.0713333Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:16:33.341398015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:16:33.344763901Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-02-16T18:16:36.382832397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35974"
level=debug ts=2026-02-16T18:16:37.072501187Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T18:16:37.139638616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:16:37.143496208Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:16:37.536177577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:16:37.539377965Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T18:16:38.071982209Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:16:38.240341149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:16:38.245045469Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:16:39.409041257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:16:39.412222105Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c67a46ea4%3A19c67a4d66e%3Ab924d97a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
ts=2026-02-16T18:16:41.121881017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T18:16:41.289773568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T18:16:41.292782529Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c67a56bbf%3A19c67a599ae%3A3ce93348?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-02-16T18:16:41.57254111Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-02-16T18:16:41.835197912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:16:41.838486525Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:16:42.338817946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:16:42.342133328Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:16:43.070915178Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:16:43.070942085Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:16:43.070950741Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:16:43.070954518Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:16:43.385079451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:16:43.39105333Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:16:43.4689507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:16:43.471845807Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:16:43.865962138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:16:43.869037852Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:16:44.455572677Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:16:46.072544134Z caller=checkpoint.go:378 msg="writing series" size="128 kB"
level=info ts=2026-02-16T18:16:46.676127652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:16:46.679515022Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T18:16:48.071642681Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:16:48.997427832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:16:49.000874704Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:16:49.576752135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:16:49.580060724Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T18:16:50.573192051Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=info ts=2026-02-16T18:16:51.321014493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:16:51.324246032Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:16:53.012195045Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T18:16:53.013866382Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:16:53.071255312Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:16:55.072719503Z caller=checkpoint.go:378 msg="writing series" size="900 B"
level=warn ts=2026-02-16T18:16:58.071811324Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:16:58.840964444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:16:58.84443431Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:16:59.572522089Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=warn ts=2026-02-16T18:17:03.071646305Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:17:04.073044979Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-02-16T18:17:07.477824617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:17:07.483341969Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:17:08.071185784Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:17:08.562386857Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:17:08.572440146Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=info ts=2026-02-16T18:17:09.575090453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:17:09.578731918Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T18:17:11.125246978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=debug ts=2026-02-16T18:17:13.054137303Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:17:13.055170058Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:17:13.071568211Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:17:13.071584726Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T18:17:13.071577643Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:17:13.071593067Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:17:13.072752589Z caller=checkpoint.go:378 msg="writing series" size="218 kB"
level=info ts=2026-02-16T18:17:14.52418177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:17:14.528017298Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T18:17:17.572219547Z caller=checkpoint.go:378 msg="writing series" size="9.5 kB"
level=warn ts=2026-02-16T18:17:18.072026451Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:17:21.157275029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:17:21.161691882Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T18:17:22.072635022Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=warn ts=2026-02-16T18:17:23.071678198Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:17:26.0431035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="587 B" avg_comp="587 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:17:26.047067503Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:17:26.572578483Z caller=checkpoint.go:378 msg="writing series" size="178 kB"
level=info ts=2026-02-16T18:17:27.434188545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:17:27.439172008Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:17:28.07117732Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:17:28.295186789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T18:17:28.298385876Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c67a56bbf%3A19c67a599ae%3A3ce93348?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T18:17:28.427634546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:17:28.430892998Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:17:30.512711318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:17:30.515902966Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T18:17:31.07297386Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=info ts=2026-02-16T18:17:31.617269372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:17:31.620914641Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:17:32.823290593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="146 B" avg_comp="146 B" total_uncomp="25 kB" avg_uncomp="25 kB" idle=1 labels="{stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:17:32.823311621Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-02-16T18:17:32.826690046Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:17:33.071460302Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:17:33.292176163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:17:33.295481845Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:17:33.410720104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:17:33.414069882Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:17:35.572904512Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
ts=2026-02-16T18:17:36.387462022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:44952"
level=info ts=2026-02-16T18:17:37.179037214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:17:37.182609065Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:17:37.549689461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:17:37.554101027Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c67a46ea4%3A19c67a4d66e%3Ab924d97a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-02-16T18:17:38.071801292Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:17:38.327660996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:17:38.334097947Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:17:38.334118485Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:17:38.334139769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:17:38.339669583Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:17:39.414081797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:17:39.41750225Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:17:40.073090915Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T18:17:40.073117038Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.1 MB" series=17
ts=2026-02-16T18:17:41.128361584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T18:17:42.489770798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:17:42.493267288Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:17:43.067925179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:17:43.070922478Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:17:43.070941513Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:17:43.070948988Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:17:43.071008721Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-02-16T18:17:43.071612445Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:17:44.572311228Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
level=debug ts=2026-02-16T18:17:45.186113499Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:17:48.003408822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:17:48.006726318Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:17:48.070984522Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:17:49.072735354Z caller=checkpoint.go:378 msg="writing series" size="286 kB"
level=info ts=2026-02-16T18:17:50.717780719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:17:50.721298279Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:17:51.995888238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:17:52.000507051Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:17:53.071084141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:17:53.233546127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:17:53.239508526Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:17:53.572871899Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=info ts=2026-02-16T18:17:55.95527952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:17:55.958617933Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:17:58.07173533Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:17:58.072837532Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=info ts=2026-02-16T18:18:02.186064951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:18:02.189803666Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T18:18:02.57316997Z caller=checkpoint.go:378 msg="writing series" size="364 kB"
level=warn ts=2026-02-16T18:18:03.071740392Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:18:04.211687798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:18:04.215454854Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T18:18:05.999470702Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:18:05.999548937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="395 kB" avg_uncomp="395 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:18:06.003744534Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-02-16T18:18:06.390084463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46456"
level=debug ts=2026-02-16T18:18:07.072659886Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-02-16T18:18:08.071716263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:18:11.131519987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=debug ts=2026-02-16T18:18:11.572280999Z caller=checkpoint.go:378 msg="writing series" size="33 kB"
level=info ts=2026-02-16T18:18:12.537943323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:18:12.543397438Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:18:13.054365124Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T18:18:13.054382775Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:18:13.070929522Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:18:13.070947684Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:18:13.07095518Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:18:13.070998735Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:18:13.374031757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:18:13.377313994Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:18:14.187270502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:18:14.190561812Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:18:15.582491657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:18:15.585837226Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T18:18:16.072366481Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T18:18:16.218794522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:18:16.222228477Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:18:18.071951515Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:18:19.641130998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:18:19.644742415Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:18:20.39569263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:18:20.398913569Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c67a46ea4%3A19c67a4d66e%3Ab924d97a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T18:18:20.572488321Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T18:18:20.892345656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:18:20.898160794Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:18:21.253853388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T18:18:21.257059713Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c67a56bbf%3A19c67a599ae%3A3ce93348?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T18:18:21.408350263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:18:21.412963042Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:18:23.071543031Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:18:23.76690144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:18:23.77059958Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:18:23.788673542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:18:23.791858405Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:18:25.072458099Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=info ts=2026-02-16T18:18:28.004702286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:18:28.007850019Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:18:28.071305532Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:18:28.776602622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:18:28.779728883Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:18:29.458779327Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:18:29.572212626Z caller=checkpoint.go:378 msg="writing series" size="101 kB"
level=warn ts=2026-02-16T18:18:33.071396279Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:18:33.135489923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:18:33.138820507Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:18:34.073116484Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T18:18:35.808615191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:18:35.811946179Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:18:36.966685951Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:18:37.557419741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:18:37.560871488Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:18:38.071440006Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:18:38.572215579Z caller=checkpoint.go:378 msg="writing series" size="180 kB"
level=debug ts=2026-02-16T18:18:38.572239683Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=13
ts=2026-02-16T18:18:41.134244207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:18:41.62379294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:18:41.627955129Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:18:43.070992509Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:18:43.071000698Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:18:43.07101954Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:18:43.071027682Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:18:43.074879621Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-02-16T18:18:43.077678631Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.005623142s
level=info ts=2026-02-16T18:18:45.59486322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:18:45.601631938Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:18:48.071630038Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:18:49.391365292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:42184"
level=info ts=2026-02-16T18:18:49.734492927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:18:49.73795305Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:18:53.071470759Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:18:54.606202009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:18:54.610871181Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:18:55.855706798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:18:55.859572883Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T18:18:58.071774395Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:19:01.025976845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:19:01.029946666Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:19:01.952597711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:19:01.957833907Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:19:02.007889354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:19:02.011036343Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:19:02.500412827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:19:02.503881389Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T18:19:03.071959806Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:19:03.18025757Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:19:04.832611251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:19:04.835882303Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:19:07.38296823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:19:07.386289393Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:19:07.453789538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:19:07.456807545Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:19:08.071750402Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:19:10.066381211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:19:10.069589905Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:19:10.814459689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T18:19:10.818894439Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c67a56bbf%3A19c67a599ae%3A3ce93348?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-02-16T18:19:11.136793217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:19:11.284549964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:19:11.28751607Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:19:13.053478271Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:19:13.054517931Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:19:13.071860106Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:19:13.071862769Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:19:13.07187896Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:19:13.071887191Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:19:13.071892332Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T18:19:13.072000542Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=debug ts=2026-02-16T18:19:13.075110231Z caller=checkpoint.go:378 msg="writing series" size="4.3 MB"
level=debug ts=2026-02-16T18:19:13.075121843Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.3 MB" series=1
level=info ts=2026-02-16T18:19:16.755367387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:19:16.760498293Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T18:19:17.311408458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:40486"
level=debug ts=2026-02-16T18:19:17.572837304Z caller=checkpoint.go:378 msg="writing series" size="266 kB"
level=warn ts=2026-02-16T18:19:18.071110604Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:19:18.203449996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:19:18.206961957Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c67a46ea4%3A19c67a4d66e%3Ab924d97a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T18:19:20.053698846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:19:20.059649752Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:19:22.072983883Z caller=checkpoint.go:378 msg="writing series" size="95 kB"
level=warn ts=2026-02-16T18:19:23.071766837Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:19:25.852269779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:19:25.855560704Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:19:26.572602836Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=debug ts=2026-02-16T18:19:26.696286572Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T18:19:28.072055913Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:19:28.852112206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:19:28.855630967Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:19:30.988368539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:19:30.99190234Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:19:31.072214137Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=debug ts=2026-02-16T18:19:31.746951308Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:19:33.071034914Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:19:35.573038536Z caller=checkpoint.go:378 msg="writing series" size="510 B"
level=info ts=2026-02-16T18:19:36.222744826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:19:36.226284751Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
ts=2026-02-16T18:19:36.395644263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46684"
level=warn ts=2026-02-16T18:19:38.071196716Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:19:40.072362118Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
ts=2026-02-16T18:19:41.139169411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T18:19:43.071937091Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:19:43.071937395Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:19:43.071964929Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:19:43.071973552Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:19:44.128201182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:19:44.13230629Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:19:44.572714223Z caller=checkpoint.go:378 msg="writing series" size="205 kB"
level=info ts=2026-02-16T18:19:44.614768822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:19:44.619354304Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:19:44.675036969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:19:44.678491697Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:19:47.98135827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:19:47.985968472Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T18:19:48.071439008Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:19:48.864636562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:19:48.867881863Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:19:49.039201584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:19:49.042356731Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T18:19:49.072405614Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
ts=2026-02-16T18:19:49.395973461Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:38680"
level=info ts=2026-02-16T18:19:49.56533557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:19:49.568613544Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:19:52.397904428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:19:52.401332433Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T18:19:53.071655766Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:19:53.572400551Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=info ts=2026-02-16T18:19:53.576489385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T18:19:53.579860523Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c67a56bbf%3A19c67a599ae%3A3ce93348?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T18:19:55.205736628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:19:55.209006588Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:19:56.455202338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:19:56.458546937Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:19:56.939615407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:19:56.943369414Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:19:58.071541575Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:19:58.072691731Z caller=checkpoint.go:378 msg="writing series" size="127 kB"
level=info ts=2026-02-16T18:19:58.895197252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:19:58.89994696Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:19:59.83676053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:19:59.840241752Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:20:02.572631198Z caller=checkpoint.go:378 msg="writing series" size="130 kB"
level=debug ts=2026-02-16T18:20:02.572662576Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=debug ts=2026-02-16T18:20:02.894055902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T18:20:03.071725495Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:20:06.131981264Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:20:07.072471701Z caller=checkpoint.go:378 msg="writing series" size="202 kB"
level=warn ts=2026-02-16T18:20:08.071932179Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:20:11.141930394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=debug ts=2026-02-16T18:20:11.57267699Z caller=checkpoint.go:378 msg="writing series" size="205 kB"
level=info ts=2026-02-16T18:20:12.378903821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:20:12.384103912Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:20:13.05334188Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:20:13.054404034Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:20:13.071622101Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:20:13.071642081Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:20:13.071650174Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:20:13.071652264Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:20:14.454922298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:20:14.460523306Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c67a46ea4%3A19c67a4d66e%3Ab924d97a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T18:20:15.138820923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:20:15.145302226Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:20:16.072996508Z caller=checkpoint.go:378 msg="writing series" size="480 kB"
level=info ts=2026-02-16T18:20:16.572208908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:20:16.575544941Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:20:17.171106114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:20:17.174508168Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:20:17.194491751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:20:17.197836841Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T18:20:18.071847929Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:20:20.573214871Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T18:20:22.095342237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:20:22.103265563Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:20:23.071859859Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:20:25.072594707Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=warn ts=2026-02-16T18:20:28.071485221Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:20:28.879909709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:20:28.883619023Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:20:29.357955549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:20:29.3612641Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:20:29.573198918Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T18:20:31.977280457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:20:31.980938952Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:20:33.071773159Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:20:34.072951016Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB"
ts=2026-02-16T18:20:36.399618913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60804"
level=warn ts=2026-02-16T18:20:38.071193766Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:20:38.573199473Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB"
level=debug ts=2026-02-16T18:20:39.0751091Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:20:40.440042835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:20:40.444194471Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T18:20:41.144842614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:20:43.07154263Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:20:43.071569803Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T18:20:43.071572988Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:20:43.071585905Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:20:43.072675488Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=info ts=2026-02-16T18:20:43.243088286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T18:20:43.246593542Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c67a56bbf%3A19c67a599ae%3A3ce93348?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T18:20:43.341001303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:20:43.344683196Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:20:43.809703477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:20:43.813438519Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:20:47.281788562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:20:47.285882229Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
ts=2026-02-16T18:20:47.316424847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:50392"
level=debug ts=2026-02-16T18:20:47.573261568Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=warn ts=2026-02-16T18:20:48.071714017Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:20:48.623483939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:20:48.626967989Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:20:49.124782919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:20:49.128533348Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:20:50.690261155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:20:50.693589526Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:20:51.50078123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:20:51.504011657Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:20:52.07264282Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T18:20:52.883849117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:20:52.886795056Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:20:53.071656066Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:20:54.199886874Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:20:54.201340157Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T18:20:54.981037246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.7 kB" avg_comp="3.7 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:20:54.984099314Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:20:56.572427088Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
level=debug ts=2026-02-16T18:20:56.572455196Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
level=info ts=2026-02-16T18:20:57.270393375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:20:57.276778164Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:20:58.071275392Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:20:59.469369446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:20:59.472901241Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c67a46ea4%3A19c67a4d66e%3Ab924d97a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T18:21:01.072256919Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=warn ts=2026-02-16T18:21:03.071812522Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:21:05.572559628Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T18:21:06.007736558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:21:06.011424845Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T18:21:06.40272121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42506"
level=warn ts=2026-02-16T18:21:08.071180519Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:21:09.02921348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:21:09.033127778Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:21:09.071337295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:21:09.074775332Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:21:09.659323576Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:21:10.072508082Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=info ts=2026-02-16T18:21:10.160809379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:21:10.16719685Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T18:21:11.14773131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=debug ts=2026-02-16T18:21:13.054191473Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:21:13.055258378Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:21:13.071658997Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:21:13.071679388Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:21:13.071687539Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:21:13.071748659Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:21:13.128846576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:21:13.132071989Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:21:13.267585301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:21:13.27058237Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T18:21:13.270599032Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:13.571169737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:21:13.574760955Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T18:21:14.572688169Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=warn ts=2026-02-16T18:21:18.071235652Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:21:19.072514297Z caller=checkpoint.go:378 msg="writing series" size="62 kB"
ts=2026-02-16T18:21:19.40158562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:54758"
level=warn ts=2026-02-16T18:21:23.071217065Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:21:23.573198475Z caller=checkpoint.go:378 msg="writing series" size="318 kB"
level=info ts=2026-02-16T18:21:25.037678341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:21:25.043471738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:21:25.045593303Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:21:25.045595278Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:21:25.045612386Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:25.045631784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:21:25.048344373Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:21:25.562764624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:21:25.566586235Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:21:28.071144704Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:21:28.072196697Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T18:21:28.788655832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:21:28.792110112Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T18:21:32.573208541Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=warn ts=2026-02-16T18:21:33.071888403Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:21:33.968516202Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:21:34.059091136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:21:34.062073382Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:21:34.298577942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:21:34.301643005Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:21:34.301660552Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:34.301680288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:21:34.304176415Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:21:34.890783487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T18:21:34.894054402Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c67a56bbf%3A19c67a599ae%3A3ce93348?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-02-16T18:21:36.405264049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51418"
level=info ts=2026-02-16T18:21:37.073215071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="917 B" avg_comp="917 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:21:37.073252986Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=error ts=2026-02-16T18:21:37.076771301Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:21:38.07155009Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:21:39.057525661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:21:39.060987392Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T18:21:39.061007415Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:39.061026842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:21:39.063837729Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-02-16T18:21:41.150330514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=debug ts=2026-02-16T18:21:41.572685541Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=debug ts=2026-02-16T18:21:41.821480793Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:21:42.62452886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:21:42.628076291Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T18:21:42.628097589Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:42.628118595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:21:42.630897191Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:21:43.071466535Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:21:43.071491882Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:21:43.071500334Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:21:43.071484042Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:21:44.026570377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:21:44.029690802Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:21:44.749553532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:21:44.753018539Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:21:44.753044938Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:44.753067387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:21:44.756186029Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:21:46.072790185Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=info ts=2026-02-16T18:21:46.718339184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:21:46.721595685Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T18:21:48.071546366Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:21:48.298499702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:21:48.301767594Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:21:48.301784614Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:48.601987131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:21:48.605188009Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-02-16T18:21:49.404560785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:51824"
level=info ts=2026-02-16T18:21:50.545284896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:21:50.548488703Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:21:50.548509112Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:50.548530274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:21:50.551109115Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:21:50.573229749Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=info ts=2026-02-16T18:21:51.126723254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:21:51.129998429Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:21:51.130018481Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:51.130043189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="182 B" avg_comp="182 B" total_uncomp="17 kB" avg_uncomp="17 kB" idle=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:21:51.132815149Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T18:21:52.481554768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:21:52.484813425Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:21:53.071778264Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:21:54.107685112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:21:54.11110673Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:21:54.11112726Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:54.111165553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:21:54.113883103Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:21:54.159121541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:21:54.163682348Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:21:54.163698953Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:54.411238559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:21:54.41578279Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:21:55.072496441Z caller=checkpoint.go:378 msg="writing series" size="900 B"
level=info ts=2026-02-16T18:21:55.562752545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:21:55.566707995Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c67a46ea4%3A19c67a4d66e%3Ab924d97a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:21:55.566729415Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:55.566751407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="327 kB" avg_comp="327 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:21:55.569752571Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:21:55.589828761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:21:55.592514643Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:21:55.925287691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:21:55.928453432Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:21:55.928473529Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:55.92849298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:21:55.931095166Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:21:56.541812672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:21:56.545091138Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:21:56.545112283Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:56.545132541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:21:56.547797844Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:21:58.071365389Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:21:58.537579901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:21:58.540970924Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:21:58.77364437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:21:58.779924099Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:21:59.572612294Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T18:22:01.764079152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:22:01.767954152Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:22:03.071299578Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:22:03.225677443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:22:03.229180695Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T18:22:03.425723231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:22:03.429249383Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:22:04.072446143Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T18:22:04.268190613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:22:04.271631205Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:22:07.0856733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:22:07.089437889Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T18:22:08.071334156Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:22:08.572212395Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=debug ts=2026-02-16T18:22:09.022636631Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:22:09.898029457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:22:09.901923769Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:22:10.074512592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:22:10.07790336Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
ts=2026-02-16T18:22:11.152896794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T18:22:11.281573182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:22:11.286506404Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:22:11.610311338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:22:11.613579312Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:22:12.290776253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:22:12.294281418Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:22:13.053948997Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:22:13.055011832Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:22:13.071227298Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:22:13.071245925Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:22:13.071253861Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:22:13.071284319Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:22:13.072443491Z caller=checkpoint.go:378 msg="writing series" size="222 kB"
level=info ts=2026-02-16T18:22:14.788007612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:22:14.791867559Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:22:16.465268912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:22:16.468542678Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:22:16.468562509Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:22:16.468584004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:22:16.471249727Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:22:17.572568957Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-02-16T18:22:18.070988962Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:22:18.239355536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:22:18.242363614Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T18:22:18.831577709Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:22:20.256329991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:22:20.259277379Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:22:20.719346064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:22:20.72267951Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T18:22:22.072353991Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=warn ts=2026-02-16T18:22:23.071299784Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:22:23.546030848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:22:23.549519593Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:22:23.860416279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:22:23.863776297Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:22:23.863794896Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:22:23.863815308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:22:23.866625738Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:22:24.692394839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="587 B" avg_comp="587 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:22:24.695727445Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:22:24.695746736Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:22:24.695770779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:22:24.698474297Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:22:26.573054716Z caller=checkpoint.go:378 msg="writing series" size="232 kB"
level=debug ts=2026-02-16T18:22:26.573082199Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=20
level=warn ts=2026-02-16T18:22:28.071183147Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:22:31.073114419Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=info ts=2026-02-16T18:22:31.659773175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:22:31.665243846Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:22:31.977948194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:22:31.981612612Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:22:33.07180654Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:22:33.821603797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T18:22:33.824978421Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c67a56bbf%3A19c67a599ae%3A3ce93348?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-02-16T18:22:33.824998788Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:22:33.82502074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="324 B" avg_comp="324 B" total_uncomp="96 kB" avg_uncomp="96 kB" idle=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:22:33.827995407Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:22:35.57274589Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T18:22:35.775356529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:22:35.778747465Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:22:36.478751287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:22:36.482730583Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:22:37.254773687Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:22:37.449531919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:22:37.453141542Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T18:22:38.0710178Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:22:38.322092819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:22:38.325404886Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:22:38.81422749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:22:38.8175811Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:22:38.975197869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:22:38.978697992Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:22:39.071824969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:22:39.077033161Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:22:40.074703201Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-02-16T18:22:40.07472871Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-02-16T18:22:40.872069757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:22:40.875705754Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
ts=2026-02-16T18:22:41.155676649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T18:22:42.275898875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:22:42.279904283Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:22:43.071663515Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:22:43.071690918Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:22:43.071700284Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:22:43.071688231Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:22:43.534042893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:22:43.537614795Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:22:44.572287259Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T18:22:45.941378949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:22:45.945090344Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T18:22:47.323356404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:43416"
level=warn ts=2026-02-16T18:22:48.071314566Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:22:49.073171447Z caller=checkpoint.go:378 msg="writing series" size="295 kB"
level=info ts=2026-02-16T18:22:49.802138475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:22:49.808452185Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:22:51.904416877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:22:51.908283736Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:22:52.294344301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:22:52.29766864Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:22:53.071925681Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:22:53.573230814Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-02-16T18:22:58.071363533Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:22:58.072456357Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=debug ts=2026-02-16T18:22:59.067117696Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:23:02.573413765Z caller=checkpoint.go:378 msg="writing series" size="384 kB"
level=warn ts=2026-02-16T18:23:03.071902814Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:23:04.711344179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:23:04.717973786Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:23:05.521332859Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:23:06.411865783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:34134"
level=debug ts=2026-02-16T18:23:07.072491923Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T18:23:07.87035407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:23:07.873888016Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:23:08.071444994Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:23:09.030585949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:23:09.034505923Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:23:10.696645232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:23:10.700265381Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:23:10.898740151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:23:10.902137117Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
ts=2026-02-16T18:23:11.158916077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=debug ts=2026-02-16T18:23:11.572258876Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=info ts=2026-02-16T18:23:12.554019749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:23:12.557353828Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:23:13.053504816Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:23:13.054580662Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:23:13.071847885Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:23:13.071876698Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:23:13.071885215Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:23:13.07190339Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:23:14.428463384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:23:14.432310138Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:23:15.205613268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:23:15.209283367Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:23:15.969558423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:23:15.972944191Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T18:23:16.072366176Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
ts=2026-02-16T18:23:17.32628567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:42890"
level=info ts=2026-02-16T18:23:17.66590918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:23:17.6692314Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T18:23:18.071501536Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:23:20.572718924Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=info ts=2026-02-16T18:23:20.787326374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:23:20.790533232Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:23:23.071539963Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:23:23.799261242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:23:23.804620628Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:23:23.87498284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:23:23.878630974Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:23:24.386200006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:23:24.38970415Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-02-16T18:23:25.07238483Z caller=checkpoint.go:378 msg="writing series" size="40 kB"
level=info ts=2026-02-16T18:23:26.390025593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:23:26.393492945Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:23:27.635320073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:23:27.638714201Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T18:23:28.071192501Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:23:29.220279754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:23:29.223769629Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:23:29.572695472Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-02-16T18:23:33.071728419Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:23:33.904499713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:23:33.908123881Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:23:34.072674112Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=debug ts=2026-02-16T18:23:36.511207574Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:23:38.0711125Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:23:38.438721388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:23:38.444081027Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T18:23:38.572616351Z caller=checkpoint.go:378 msg="writing series" size="180 kB"
level=debug ts=2026-02-16T18:23:38.572642543Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=13
level=info ts=2026-02-16T18:23:39.579946475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:23:39.586932387Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:23:40.982427998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:23:40.985802003Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T18:23:41.161517237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T18:23:43.071534086Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:23:43.071565677Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:23:43.071574062Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:23:43.071627898Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:23:43.078504857Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-02-16T18:23:43.079832552Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.00769922s
level=debug ts=2026-02-16T18:23:46.581144604Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T18:23:48.071670103Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:23:49.411905934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:33734"
level=info ts=2026-02-16T18:23:50.120881057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:23:50.124080886Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:23:53.071866996Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:23:58.071214472Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:23:58.457618242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:23:58.461248179Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:23:58.706772853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:23:58.710136027Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:24:00.29358331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:24:00.296804413Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T18:24:03.07169973Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:24:03.13386485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:24:03.137603247Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:24:04.410074367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:24:04.413988004Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:24:04.565298048Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:24:05.21365387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:24:05.217300273Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T18:24:06.41593167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46180"
level=info ts=2026-02-16T18:24:07.546733537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:24:07.549830828Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:24:07.717353945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:24:07.720632462Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:24:07.72065139Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:24:07.720673735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:24:07.723536333Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:24:08.071451305Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:24:08.502969532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:24:08.507186122Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:24:10.161569362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:24:10.165401966Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-02-16T18:24:11.164286009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:24:11.354194154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:24:11.357427451Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-02-16T18:24:13.054200097Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:24:13.055247698Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:24:13.071313472Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T18:24:13.071358288Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:24:13.071363393Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:24:13.071382672Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:24:13.071390676Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:24:13.071444078Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=debug ts=2026-02-16T18:24:13.075861494Z caller=checkpoint.go:378 msg="writing series" size="5.0 MB"
level=debug ts=2026-02-16T18:24:13.075875851Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.0 MB" series=1
level=info ts=2026-02-16T18:24:15.405042471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:24:15.408741852Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:24:16.29980192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:24:16.303342114Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:24:16.78253837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:24:16.785718354Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:24:17.498199062Z caller=checkpoint.go:378 msg="writing series" size="367 kB"
level=warn ts=2026-02-16T18:24:18.071648546Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:24:18.271514164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:24:18.276240499Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:24:19.151348659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:24:19.154845545Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T18:24:19.414492069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:40510"
level=info ts=2026-02-16T18:24:20.475423367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:24:20.478920518Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:24:21.924683826Z caller=checkpoint.go:378 msg="writing series" size="114 kB"
level=warn ts=2026-02-16T18:24:23.071321711Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:24:25.749016079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:24:25.755704671Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:24:25.755726469Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:24:25.755749956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:24:25.762513029Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:24:26.350656016Z caller=checkpoint.go:378 msg="writing series" size="51 kB"
level=info ts=2026-02-16T18:24:26.969429497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:24:26.972992205Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:24:28.071688393Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:24:29.037742285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:24:29.042268893Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:24:29.486891141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:24:29.490755505Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T18:24:30.777122064Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=debug ts=2026-02-16T18:24:33.015250299Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:24:33.015312857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="190 B" avg_comp="190 B" total_uncomp="62 kB" avg_uncomp="62 kB" idle=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:24:33.019175638Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T18:24:33.071954389Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:24:35.203296422Z caller=checkpoint.go:378 msg="writing series" size="510 B"
level=debug ts=2026-02-16T18:24:37.295507691Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:24:37.297377069Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:24:38.071568535Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:24:38.995750517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:24:39.000982151Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:24:39.629700041Z caller=checkpoint.go:378 msg="writing series" size="90 kB"
ts=2026-02-16T18:24:41.16692031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:24:43.071536286Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:24:43.071575779Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:24:43.071589766Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:24:43.071658986Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:24:44.055959666Z caller=checkpoint.go:378 msg="writing series" size="349 kB"
level=info ts=2026-02-16T18:24:44.626082134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:24:44.630943316Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:24:45.545447706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:24:45.552143403Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:24:47.321591909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:24:47.325427907Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T18:24:47.332332267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:55164"
level=warn ts=2026-02-16T18:24:48.071665572Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:24:48.481861717Z caller=checkpoint.go:378 msg="writing series" size="63 kB"
level=debug ts=2026-02-16T18:24:48.481888208Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-02-16T18:24:48.495957235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:24:48.49966169Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:24:51.310801022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:24:51.314435886Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T18:24:52.908654604Z caller=checkpoint.go:378 msg="writing series" size="125 kB"
level=warn ts=2026-02-16T18:24:53.071406696Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:24:53.545979154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:24:53.549477114Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:24:54.971209011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:24:54.974899645Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:24:57.028871917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:24:57.032839659Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:24:57.334966256Z caller=checkpoint.go:378 msg="writing series" size="218 kB"
level=info ts=2026-02-16T18:24:57.679901242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:24:57.684041109Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:24:58.07099317Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:25:00.256688783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:25:00.260382059Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:25:00.936209108Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:25:01.187936112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:25:01.191490074Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T18:25:01.760552173Z caller=checkpoint.go:378 msg="writing series" size="200 kB"
level=info ts=2026-02-16T18:25:01.995026401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:25:01.998586532Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:25:02.006681251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:25:02.010138106Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:25:02.010180292Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:25:02.010205881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:25:02.013363497Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:25:03.071616434Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:25:06.187176435Z caller=checkpoint.go:378 msg="writing series" size="231 kB"
level=info ts=2026-02-16T18:25:07.063068236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:25:07.066779484Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:25:08.071506865Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:25:08.480885763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:25:08.484845019Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T18:25:09.112826469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:25:09.117619402Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:25:10.613235463Z caller=checkpoint.go:378 msg="writing series" size="417 kB"
level=debug ts=2026-02-16T18:25:10.613262027Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=5
ts=2026-02-16T18:25:11.169925535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T18:25:11.318776092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:25:11.32230771Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:25:12.225753533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:25:12.2294201Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:25:12.752002728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:25:12.755653416Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:25:13.053978233Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:25:13.055048324Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:25:13.071136507Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:25:13.071180351Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:25:13.071189513Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:25:13.071204615Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:25:14.136294633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:25:14.140317624Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:25:14.27376631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:25:14.278523256Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:25:14.464274434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:25:14.467677652Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:25:15.03927794Z caller=checkpoint.go:378 msg="writing series" size="504 kB"
level=warn ts=2026-02-16T18:25:18.071185832Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:25:19.420193838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:41404"
level=debug ts=2026-02-16T18:25:19.465308005Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T18:25:20.213394322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:25:20.220335217Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:25:20.754916318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:25:20.758984274Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:25:23.070967308Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:25:23.696077855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:25:23.699301823Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:25:23.892009793Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=debug ts=2026-02-16T18:25:24.292585345Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:25:26.489737947Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:25:28.071987517Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:25:28.317926682Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T18:25:30.630564191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:25:30.634873144Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:25:32.182045848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:25:32.185483877Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:25:32.743933271Z caller=checkpoint.go:378 msg="writing series" size="4.7 kB"
level=warn ts=2026-02-16T18:25:33.071215622Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:25:36.88371268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:25:36.888261077Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T18:25:37.170015Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB"
level=warn ts=2026-02-16T18:25:38.071323694Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:25:38.525605318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:25:38.52930383Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-02-16T18:25:41.172563952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=debug ts=2026-02-16T18:25:41.596986028Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T18:25:42.767975541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:25:42.777677662Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:25:43.071778137Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:25:43.071778752Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:25:43.071805743Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:25:43.071814057Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:25:46.02324178Z caller=checkpoint.go:378 msg="writing series" size="321 kB"
level=info ts=2026-02-16T18:25:46.193460793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:25:46.197105725Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:25:47.004277202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:25:47.008141779Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T18:25:47.336708592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:34156"
level=warn ts=2026-02-16T18:25:48.071623151Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:25:49.423431291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:44426"
level=info ts=2026-02-16T18:25:49.5379017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:25:49.542740491Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:25:49.791571706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:25:49.795675157Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:25:50.449423171Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-02-16T18:25:53.070995678Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:25:54.875365675Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
level=debug ts=2026-02-16T18:25:54.87540366Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-02-16T18:25:57.229369103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:25:57.234589791Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T18:25:58.071718714Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:25:58.077763001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:25:58.081526441Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:25:58.688190675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:25:58.692100226Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:25:59.301774716Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=info ts=2026-02-16T18:26:00.218597499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:26:00.222726337Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:26:00.422285575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:26:00.426499528Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T18:26:01.420230776Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:26:01.420285568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="354 B" avg_comp="354 B" total_uncomp="354 B" avg_uncomp="354 B" idle=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:26:01.424018988Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:26:03.070952923Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:26:03.20027455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:26:03.203688487Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T18:26:03.64295666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:26:03.646488413Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:26:03.689864369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:26:03.693387791Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:26:03.727511236Z caller=checkpoint.go:378 msg="writing series" size="118 kB"
level=debug ts=2026-02-16T18:26:03.994700958Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T18:26:08.071696751Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:26:08.091738116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:26:08.095610668Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:26:08.153847673Z caller=checkpoint.go:378 msg="writing series" size="4.0 kB"
ts=2026-02-16T18:26:11.175535684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T18:26:12.432180844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:26:12.435564677Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T18:26:12.579969306Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=debug ts=2026-02-16T18:26:13.053556239Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:26:13.054617599Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:26:13.071110741Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:26:13.071109225Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:26:13.071138229Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:26:13.07114633Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:26:13.886477178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.7 kB" avg_comp="3.7 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:26:13.890764902Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:26:14.269718427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:26:14.274739321Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:26:17.006285751Z caller=checkpoint.go:378 msg="writing series" size="887 B"
ts=2026-02-16T18:26:17.340306131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:45168"
level=warn ts=2026-02-16T18:26:18.071618488Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:26:19.426832589Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:44788"
level=info ts=2026-02-16T18:26:19.472385809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:26:19.476015995Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:26:19.62274016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:26:19.631016076Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:26:21.185331074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="453 B" avg_comp="453 B" total_uncomp="354 B" avg_uncomp="354 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:26:21.188903338Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:26:21.432511013Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-02-16T18:26:23.07128012Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:26:25.859027121Z caller=checkpoint.go:378 msg="writing series" size="236 kB"
level=info ts=2026-02-16T18:26:26.438407054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:26:26.443105816Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:26:28.071389782Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:26:29.981010091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:26:29.984895837Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:26:30.284909854Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=debug ts=2026-02-16T18:26:32.85529414Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:26:33.071443428Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:26:33.615433407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:26:33.619264922Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:26:33.901432132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:26:33.9048169Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T18:26:34.711350837Z caller=checkpoint.go:378 msg="writing series" size="63 kB"
ts=2026-02-16T18:26:36.425335131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39198"
level=info ts=2026-02-16T18:26:36.884091752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:26:36.887965265Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:26:37.054559958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:26:37.057763335Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T18:26:38.071569111Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:26:39.137679416Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T18:26:40.572378028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:26:40.576146597Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:26:41.017263573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:26:41.02061452Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T18:26:41.178226866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T18:26:41.803957827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="917 B" avg_comp="917 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:26:41.807436725Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:26:42.826249287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:26:42.830381728Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T18:26:43.0716015Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:26:43.071599272Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:26:43.071631614Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:26:43.071640007Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:26:43.564199776Z caller=checkpoint.go:378 msg="writing series" size="6.8 kB"
level=info ts=2026-02-16T18:26:44.100206866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:26:44.106042815Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-02-16T18:26:47.342907212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:58740"
level=debug ts=2026-02-16T18:26:47.990340152Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=warn ts=2026-02-16T18:26:48.071610528Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:26:48.813650672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:26:48.817242175Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:26:49.182602921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:26:49.186069145Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:26:51.187665876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="471 B" avg_comp="471 B" total_uncomp="354 B" avg_uncomp="354 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:26:51.191077743Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:26:52.416338816Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=warn ts=2026-02-16T18:26:53.071620393Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:26:53.677738227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:26:53.681458992Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:26:53.861068156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:26:53.864702963Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:26:56.842590495Z caller=checkpoint.go:378 msg="writing series" size="900 B"
level=debug ts=2026-02-16T18:26:56.900079781Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:26:57.617829836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:26:57.620865324Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-02-16T18:26:58.071567892Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:26:59.751904383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:27:01.268418546Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T18:27:02.346356454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:27:02.351247824Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:27:03.071642051Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:27:05.038510728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:27:05.042208107Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T18:27:05.69507834Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
ts=2026-02-16T18:27:06.428452226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33458"
level=info ts=2026-02-16T18:27:07.596200642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:27:07.599855238Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:27:08.046062108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:27:08.050770609Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:27:08.071115896Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:27:10.121208042Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
ts=2026-02-16T18:27:11.181114343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=debug ts=2026-02-16T18:27:13.053519946Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:27:13.054560752Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T18:27:13.071060117Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:27:13.071067441Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:27:13.07108909Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:27:13.071097624Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:27:14.547449138Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-02-16T18:27:15.498269344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:27:15.510514627Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:27:17.559926955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:27:17.563587197Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T18:27:18.071492208Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:27:18.422613133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="587 B" avg_comp="587 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:27:18.426195841Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:27:18.973722315Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=info ts=2026-02-16T18:27:19.243028408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:27:19.246990981Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:27:21.297949726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:27:21.30213229Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:27:22.264768284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:27:22.268630979Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:27:23.071123796Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:27:23.400408641Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=info ts=2026-02-16T18:27:23.413531986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:27:23.417190537Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:27:24.02999151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:27:24.03342159Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:27:27.734782252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:27:27.738266022Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:27:27.826800406Z caller=checkpoint.go:378 msg="writing series" size="397 kB"
level=debug ts=2026-02-16T18:27:27.826824462Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=21
level=warn ts=2026-02-16T18:27:28.071849123Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:27:29.10208312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:27:29.10577032Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:27:29.692813256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:27:29.696384623Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:27:30.490671113Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:27:30.541733635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:27:30.545503316Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T18:27:32.252293154Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=warn ts=2026-02-16T18:27:33.071919617Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:27:33.837779311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:27:34.778429504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:27:34.781947524Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:27:36.678309852Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T18:27:36.97948932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="471 B" avg_comp="471 B" total_uncomp="354 B" avg_uncomp="354 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:27:36.982977335Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:27:38.071732573Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:27:39.194770665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:27:39.198908435Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:27:39.750217025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:27:39.754335142Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-02-16T18:27:41.105908575Z caller=checkpoint.go:378 msg="writing series" size="1.5 MB"
level=debug ts=2026-02-16T18:27:41.105941056Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=3
ts=2026-02-16T18:27:41.184211346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:27:41.280457609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:27:41.286621752Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T18:27:43.070976895Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:27:43.070990128Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:27:43.071016249Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:27:43.071024991Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:27:44.597017313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:27:44.600942013Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:27:45.531718778Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=info ts=2026-02-16T18:27:45.85469905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:27:45.860087493Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:27:45.950356687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:27:45.953877323Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T18:27:45.95389592Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:27:45.953919202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:27:45.957284485Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T18:27:48.071171197Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:27:48.847921795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:27:48.851916668Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:27:49.961646511Z caller=checkpoint.go:378 msg="writing series" size="7.0 MB"
level=debug ts=2026-02-16T18:27:49.961669842Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.1 MB" series=2
level=warn ts=2026-02-16T18:27:53.07154701Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:27:54.383726881Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-02-16T18:27:58.071202983Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:27:58.809936658Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=debug ts=2026-02-16T18:28:00.960615821Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:28:01.184283869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:28:01.192306133Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:28:01.822965968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:28:01.826695765Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:28:03.07163909Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:28:03.23641617Z caller=checkpoint.go:378 msg="writing series" size="675 kB"
level=info ts=2026-02-16T18:28:04.033478193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:28:04.039174648Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
ts=2026-02-16T18:28:06.432337748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46460"
level=debug ts=2026-02-16T18:28:07.662751544Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-02-16T18:28:08.071884653Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:28:08.156225092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:28:08.16114372Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:28:08.161186463Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:28:08.161218209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:28:08.164694736Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:28:11.007538357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:28:11.011745851Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T18:28:11.011774075Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:28:11.011802653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:28:11.015284461Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
ts=2026-02-16T18:28:11.186910674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:28:11.213505305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:28:11.221371675Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:28:12.088867902Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
level=debug ts=2026-02-16T18:28:13.053683568Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:28:13.05473493Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:28:13.070967384Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:28:13.070991647Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:28:13.070999698Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:28:13.071006206Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:28:13.578175147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:28:13.58193495Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:28:14.318191649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:28:14.32160989Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:28:15.199391849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:28:15.203627375Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:28:15.203649187Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:28:15.203674637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:28:15.207520168Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:28:15.516559641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:28:15.520229454Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:28:15.520249571Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:28:15.520271809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:28:15.523540242Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T18:28:16.514475777Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-02-16T18:28:17.216389792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:28:17.221548445Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T18:28:17.349564896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:53446"
level=warn ts=2026-02-16T18:28:18.071494409Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:28:19.476661566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:28:19.480562059Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:28:19.575751532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:28:19.579317869Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:28:19.579338369Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:28:19.579363991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:28:19.583296206Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:28:19.63550228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:28:19.639252889Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T18:28:19.639279637Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:28:19.882225025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:28:19.889202735Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:28:20.930734565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:28:20.934638196Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T18:28:20.940718282Z caller=checkpoint.go:378 msg="writing series" size="72 kB"
level=warn ts=2026-02-16T18:28:23.071109762Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:28:23.108278519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:28:23.111973148Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:28:23.111993304Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:28:23.112015712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:28:23.115965303Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:28:23.957453197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:28:23.960898714Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:28:23.960920483Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:28:23.960941445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:28:23.964406859Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:28:25.367730502Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-02-16T18:28:25.451983447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:28:25.455860158Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:28:28.071109884Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:28:28.22648703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:28:28.230060213Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:28:28.321725779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="471 B" avg_comp="471 B" total_uncomp="354 B" avg_uncomp="354 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:28:28.325402072Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:28:29.793755524Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=info ts=2026-02-16T18:28:30.520502492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:28:30.524368698Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:28:31.033000807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:28:31.036481427Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:28:31.06176456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:28:31.064889794Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:28:32.429569532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:28:32.433669911Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:28:32.538864122Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:28:32.540713471Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T18:28:33.008449282Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:28:33.071220887Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:28:34.149364017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:28:34.154222919Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:28:34.154252624Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:28:34.154283116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:28:34.158058671Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:28:34.187126957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:28:34.190766094Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:28:34.219953067Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T18:28:34.542527098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:28:34.546349343Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
ts=2026-02-16T18:28:36.435443581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60234"
level=warn ts=2026-02-16T18:28:38.071009462Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:28:38.161337815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:28:38.165630502Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:28:38.165654384Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:28:38.165683182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:28:38.168929583Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:28:38.646193246Z caller=checkpoint.go:378 msg="writing series" size="181 kB"
level=debug ts=2026-02-16T18:28:38.646218288Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=11
level=info ts=2026-02-16T18:28:39.426163103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:28:39.430020312Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:28:40.488143232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:28:40.492325892Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T18:28:41.190414149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:28:42.086336669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:28:42.089817367Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:28:43.071004657Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:28:43.071032318Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:28:43.071040953Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:28:43.071079299Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:28:43.201310129Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-02-16T18:28:43.20269188Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.131134609s
level=info ts=2026-02-16T18:28:43.29605529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:28:43.300057117Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:28:43.30008406Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:28:43.300109583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:28:43.303171257Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:28:44.44969159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:28:44.453301964Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:28:44.873786985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:28:44.877656801Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:28:44.877684914Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:28:44.877714778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:28:44.882182506Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
ts=2026-02-16T18:28:47.352915052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:34678"
level=warn ts=2026-02-16T18:28:48.071997625Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:28:52.129096706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:28:52.134413085Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:28:52.661000464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:28:52.667059325Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:28:53.071540406Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:28:55.149742444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:28:55.153111018Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:28:55.231199271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:28:55.234844699Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:28:55.522780883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:28:55.528526977Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:28:56.523345883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:28:56.527497062Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:28:57.262703499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:28:57.267657986Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:28:57.267678084Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:28:57.26770044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:28:57.272332839Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:28:57.785276519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:28:57.788787948Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:28:58.071831503Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:29:00.341398167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:29:00.34518159Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:29:02.013427497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:29:02.016906432Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:29:02.016924836Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:29:02.016945278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:29:02.020212318Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:29:02.330126525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:29:02.338078352Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T18:29:03.071889736Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:29:04.788069593Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:29:05.410746249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:29:05.415550333Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T18:29:06.438386007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42196"
level=info ts=2026-02-16T18:29:06.806912042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:29:06.810560226Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:29:07.29809569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:29:07.301405666Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T18:29:08.072127192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:29:08.958276408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:29:08.962208068Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:29:08.962228574Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:29:08.962250641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:29:08.965508558Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:29:10.57270495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:29:11.193776753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:29:11.753954608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:29:11.758490996Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:29:12.835433382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:29:12.843980246Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:29:13.05371659Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:29:13.054774528Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:29:13.071462028Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:29:13.071494204Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:29:13.071505098Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:29:13.071516248Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-02-16T18:29:13.07150579Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:29:13.071645087Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=debug ts=2026-02-16T18:29:13.074952973Z caller=checkpoint.go:378 msg="writing series" size="5.0 MB"
level=debug ts=2026-02-16T18:29:13.074969129Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.0 MB" series=1
level=info ts=2026-02-16T18:29:13.510222132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:29:13.515710303Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:29:13.706286829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="471 B" avg_comp="471 B" total_uncomp="354 B" avg_uncomp="354 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:29:13.709913267Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:29:16.929247045Z caller=checkpoint.go:378 msg="writing series" size="301 kB"
ts=2026-02-16T18:29:17.356066062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:59852"
level=info ts=2026-02-16T18:29:17.560529918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:29:17.56464678Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T18:29:18.071431895Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:29:18.279742385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:29:18.283648497Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:29:18.424097731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:29:18.432497974Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:29:18.847649376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:29:18.853235927Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:29:20.665200652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:29:20.6695245Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:29:20.786529086Z caller=checkpoint.go:378 msg="writing series" size="114 kB"
level=info ts=2026-02-16T18:29:22.36099855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:29:22.368432701Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:29:23.071873107Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:29:24.348085811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:29:24.351885345Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:29:24.351908512Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:29:24.351940168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:29:24.354955123Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:29:24.644232238Z caller=checkpoint.go:378 msg="writing series" size="66 kB"
level=info ts=2026-02-16T18:29:24.970292032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:29:24.973874639Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:29:24.973894002Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:29:24.973914073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:29:24.976592938Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:29:27.695690964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:29:27.699325012Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:29:28.071803011Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:29:28.50114045Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=debug ts=2026-02-16T18:29:32.358024479Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=warn ts=2026-02-16T18:29:33.071519076Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:29:33.899426825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:29:33.903227261Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:29:36.21539211Z caller=checkpoint.go:378 msg="writing series" size="510 B"
level=debug ts=2026-02-16T18:29:37.101953892Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:29:38.071514423Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:29:40.072850305Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=info ts=2026-02-16T18:29:40.445673941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" idle=1 full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:29:40.450401947Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:29:40.527634237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:29:40.53317324Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:29:41.065858848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:29:41.069734813Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:29:41.127826762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.3 kB" avg_uncomp="7.3 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:29:41.131232011Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
ts=2026-02-16T18:29:41.19646441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T18:29:42.075195588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:29:42.079240546Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:29:43.071915091Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:29:43.071945244Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:29:43.071953902Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:29:43.071972639Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:29:43.746724749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:29:43.750062534Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:29:43.929665924Z caller=checkpoint.go:378 msg="writing series" size="917 B"
level=debug ts=2026-02-16T18:29:45.487124405Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:29:47.358829298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49962"
level=debug ts=2026-02-16T18:29:47.786556561Z caller=checkpoint.go:378 msg="writing series" size="251 kB"
level=warn ts=2026-02-16T18:29:48.071420833Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:29:49.601902126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:29:49.605296148Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:29:49.881212285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="20 MB" avg_uncomp="20 MB" synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:29:49.887393679Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:29:51.605018533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:29:51.608584433Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T18:29:51.643706247Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=warn ts=2026-02-16T18:29:53.071228753Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:29:55.414897542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:29:55.420011856Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:29:55.500537492Z caller=checkpoint.go:378 msg="writing series" size="141 kB"
level=debug ts=2026-02-16T18:29:55.500569761Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=11
level=info ts=2026-02-16T18:29:57.213007953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:29:57.217064481Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T18:29:58.071632032Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:29:59.203282424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:29:59.206745821Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:29:59.35763782Z caller=checkpoint.go:378 msg="writing series" size="95 kB"
level=info ts=2026-02-16T18:29:59.982724209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.9 kB" avg_uncomp="4.4 kB" idle=1 full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:29:59.986140688Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:30:00.160743776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:30:00.164311752Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:30:00.259643616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:30:00.263093609Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:30:00.560901509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:30:00.564698782Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:30:01.872615326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:30:01.876091496Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:30:03.07139122Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:30:03.214903302Z caller=checkpoint.go:378 msg="writing series" size="340 kB"
level=info ts=2026-02-16T18:30:03.217810775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:30:03.221261383Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:30:03.597652719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="471 B" avg_comp="471 B" total_uncomp="354 B" avg_uncomp="354 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:30:03.601020416Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:30:04.683870026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:30:04.687186949Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:30:06.267660264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:30:06.271203769Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:30:07.072785798Z caller=checkpoint.go:378 msg="writing series" size="383 kB"
level=info ts=2026-02-16T18:30:07.504299993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:30:07.50798259Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:30:07.729798474Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:30:08.07134062Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:30:10.930083211Z caller=checkpoint.go:378 msg="writing series" size="395 kB"
level=debug ts=2026-02-16T18:30:10.93011775Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-02-16T18:30:11.103490792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:30:11.108271644Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
ts=2026-02-16T18:30:11.199654197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=debug ts=2026-02-16T18:30:13.05366085Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:30:13.054724335Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T18:30:13.05601872Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-02-16T18:30:13.056060539Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1968073 n=1
level=debug ts=2026-02-16T18:30:13.056111663Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1968073 n=1
level=info ts=2026-02-16T18:30:13.071247599Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:30:13.071278024Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:30:13.071288135Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:30:13.071300106Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:30:13.559368932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:30:13.56320111Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T18:30:14.786697183Z caller=checkpoint.go:378 msg="writing series" size="601 kB"
level=warn ts=2026-02-16T18:30:18.071952653Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:30:18.644359767Z caller=checkpoint.go:378 msg="writing series" size="6.4 kB"
level=info ts=2026-02-16T18:30:20.82745436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:30:20.831692401Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:30:22.501348576Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
level=warn ts=2026-02-16T18:30:23.071683758Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:30:26.357944361Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T18:30:26.965695687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:30:26.972171256Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:30:28.071396482Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:30:30.215306608Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=warn ts=2026-02-16T18:30:33.070998584Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:30:34.072801014Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=info ts=2026-02-16T18:30:35.642981157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:30:35.647164728Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:30:35.647189541Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:30:35.647213661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:30:35.650610563Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T18:30:36.445127706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:45746"
level=info ts=2026-02-16T18:30:37.233678586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" idle=1 full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:30:37.238220171Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:30:37.929600767Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB"
level=warn ts=2026-02-16T18:30:38.071493416Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:30:38.15275206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:30:38.159984569Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:30:38.828690837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:30:38.833477583Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:30:40.899200433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:30:40.903563897Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:30:40.989775947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:30:40.993260395Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
ts=2026-02-16T18:30:41.20279207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=debug ts=2026-02-16T18:30:41.786845994Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=debug ts=2026-02-16T18:30:41.808073937Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:30:41.970474112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:30:41.974488007Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:30:43.070978778Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:30:43.07098476Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:30:43.071011829Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:30:43.071020807Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:30:44.061272226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="471 B" avg_comp="471 B" total_uncomp="354 B" avg_uncomp="354 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:30:44.065012023Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:30:44.687811896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:30:44.69267886Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:30:45.643529799Z caller=checkpoint.go:378 msg="writing series" size="2.3 kB"
level=info ts=2026-02-16T18:30:45.658858104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:30:45.665117624Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:30:45.83920848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.7 kB" avg_comp="3.7 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:30:45.844014088Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:30:47.168690136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:30:47.184823628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:30:47.187970629Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:30:47.835299871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:30:47.839401654Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:30:48.071603641Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:30:49.504402935Z caller=checkpoint.go:378 msg="writing series" size="327 kB"
level=debug ts=2026-02-16T18:30:49.504436716Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=10
level=info ts=2026-02-16T18:30:51.694644468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:30:51.698285924Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:30:52.418230555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:30:52.429824491Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:30:53.012827689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.9 kB" avg_uncomp="4.4 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:30:53.02782824Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T18:30:53.071660704Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:30:53.358454148Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T18:30:54.168205732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:30:54.171619753Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:30:55.018254193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:30:55.027691758Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:30:55.081200364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:30:55.084560987Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:30:56.155444269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:30:56.158856598Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:30:56.158878756Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:30:56.158901095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:30:56.161687937Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:30:56.388666533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:30:56.393217578Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:30:57.220395299Z caller=checkpoint.go:378 msg="writing series" size="242 kB"
level=info ts=2026-02-16T18:30:57.586665243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:30:57.594951742Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:30:58.071538642Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:31:01.072406252Z caller=checkpoint.go:378 msg="writing series" size="7.2 kB"
level=warn ts=2026-02-16T18:31:03.07172972Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:31:04.929304825Z caller=checkpoint.go:378 msg="writing series" size="118 kB"
level=info ts=2026-02-16T18:31:07.53819153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:31:07.542787865Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:31:08.072063688Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:31:08.118247412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:31:08.127532943Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:31:08.709316067Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:31:08.786847416Z caller=checkpoint.go:378 msg="writing series" size="222 kB"
level=info ts=2026-02-16T18:31:10.15454929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:31:10.158258476Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
ts=2026-02-16T18:31:11.207731126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=debug ts=2026-02-16T18:31:12.644062104Z caller=checkpoint.go:378 msg="writing series" size="4.0 kB"
level=debug ts=2026-02-16T18:31:13.054046874Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:31:13.055089347Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:31:13.071457985Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:31:13.07148772Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:31:13.071495909Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:31:13.071496125Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:31:16.500775915Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=warn ts=2026-02-16T18:31:18.071712324Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:31:18.854749943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:31:18.859913956Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:31:20.35852444Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=warn ts=2026-02-16T18:31:23.071846864Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:31:24.216862948Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-02-16T18:31:24.476416873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:31:24.481775774Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:31:28.071266118Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:31:28.072524895Z caller=checkpoint.go:378 msg="writing series" size="364 kB"
level=info ts=2026-02-16T18:31:30.982376595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:31:30.988235878Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:31:31.929646252Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-02-16T18:31:33.071755226Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:31:33.669290462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:31:33.673338946Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:31:34.635372832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:31:34.639384502Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:31:34.639413982Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:31:34.639444743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:31:34.642890693Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:31:35.061600648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:31:35.068118844Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:31:35.068143882Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:31:35.068185818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:31:35.074663168Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:31:35.787100837Z caller=checkpoint.go:378 msg="writing series" size="72 kB"
level=debug ts=2026-02-16T18:31:35.78713779Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
level=info ts=2026-02-16T18:31:37.054203444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:31:37.058014852Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:31:37.147274138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:31:37.150786155Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:31:37.741823776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:31:37.746131231Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:31:37.749170662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="453 B" avg_comp="453 B" total_uncomp="354 B" avg_uncomp="354 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:31:37.7528652Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:31:38.054090609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T18:31:38.071713695Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:31:38.222193109Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:31:39.617615839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.9 kB" avg_uncomp="4.4 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:31:39.62118807Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T18:31:39.643345159Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T18:31:40.636610719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:31:40.640744251Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:31:40.701102892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:31:40.704994971Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
ts=2026-02-16T18:31:41.213369376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:31:41.314544435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:31:41.318627941Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:31:42.482077018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:31:42.485524837Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:31:42.966901659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:31:42.970340438Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:31:43.070920616Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:31:43.070950812Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:31:43.070958686Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:31:43.070960056Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:31:43.501306175Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=info ts=2026-02-16T18:31:45.757262962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:31:45.763588506Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:31:46.218765426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="917 B" avg_comp="917 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:31:46.222097929Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:31:46.235178852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:31:46.238383854Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:31:46.64549651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:31:46.649073162Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T18:31:47.358024351Z caller=checkpoint.go:378 msg="writing series" size="129 kB"
ts=2026-02-16T18:31:47.366504144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:35268"
level=info ts=2026-02-16T18:31:47.9166515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:31:47.920040715Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:31:48.071909443Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:31:51.214968231Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=info ts=2026-02-16T18:31:51.27735679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:31:51.282843764Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:31:51.738093142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:31:51.742005859Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:31:53.071350753Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:31:53.378995362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:31:53.382579236Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:31:55.072806557Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=warn ts=2026-02-16T18:31:58.071559972Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:31:58.929026817Z caller=checkpoint.go:378 msg="writing series" size="900 B"
level=info ts=2026-02-16T18:32:01.467779882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:32:01.473290225Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:32:02.7870366Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=warn ts=2026-02-16T18:32:03.071405269Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:32:04.371948467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:32:04.375755139Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:32:05.947807802Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:32:05.947877418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="35 kB" avg_uncomp="35 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:32:05.951621804Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:32:06.64421242Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T18:32:08.071808609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:32:10.501439639Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
ts=2026-02-16T18:32:11.216617222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:32:12.970009388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:32:12.975502147Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:32:13.05386718Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:32:13.054923284Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:32:13.071465316Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:32:13.07149533Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:32:13.071505647Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:32:13.071493379Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:32:14.357780662Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-02-16T18:32:18.071287685Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:32:18.215671831Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=info ts=2026-02-16T18:32:18.722777779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="471 B" avg_comp="471 B" total_uncomp="354 B" avg_uncomp="354 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:32:18.726909975Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:32:18.72693357Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:32:18.726958128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="471 B" avg_comp="471 B" total_uncomp="354 B" avg_uncomp="354 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:32:18.729993462Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:32:18.999730407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:32:19.006668984Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T18:32:19.44581262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:40072"
level=info ts=2026-02-16T18:32:20.384722357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:32:20.388605727Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:32:22.07237986Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=info ts=2026-02-16T18:32:22.863800537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:32:22.868607146Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T18:32:23.071726184Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:32:23.201957131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:32:23.205533298Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:32:24.078987767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:32:24.082590124Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:32:24.294486549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.9 kB" avg_uncomp="4.4 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:32:24.298767615Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:32:25.427527356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:32:25.43137343Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T18:32:25.929890305Z caller=checkpoint.go:378 msg="writing series" size="205 kB"
level=info ts=2026-02-16T18:32:26.523661973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:32:26.527553911Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:32:27.801937111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:32:27.805512129Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:32:27.908793787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:32:27.91241469Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T18:32:28.071215698Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:32:28.395077418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:32:28.399024384Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:32:28.447645473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:32:28.451343578Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:32:29.786607431Z caller=checkpoint.go:378 msg="writing series" size="480 kB"
level=debug ts=2026-02-16T18:32:29.78663418Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=14
level=info ts=2026-02-16T18:32:29.984774264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:32:29.988441476Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T18:32:31.166312323Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:32:31.919367071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="587 B" avg_comp="587 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:32:31.923720941Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:32:33.071677999Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:32:33.64345247Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=info ts=2026-02-16T18:32:35.173173586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:32:35.176834317Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:32:35.333208819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="471 B" avg_comp="471 B" total_uncomp="354 B" avg_uncomp="354 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:32:35.336544417Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:32:35.613365977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:32:35.616820785Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T18:32:36.187105131Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:32:36.188949213Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T18:32:37.500826042Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T18:32:37.841949765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:32:37.845354722Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:32:38.071236228Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:32:39.702435377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:32:39.705897034Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:32:40.404906652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:32:40.408407003Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
ts=2026-02-16T18:32:41.219793389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=debug ts=2026-02-16T18:32:41.358939295Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-02-16T18:32:41.358969584Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-02-16T18:32:43.071628017Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:32:43.071665607Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:32:43.071676914Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:32:43.071698841Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:32:43.450922639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:32:43.454726216Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:32:45.215282429Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=info ts=2026-02-16T18:32:47.805096329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:32:47.809123356Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T18:32:48.071187845Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:32:49.080822846Z caller=checkpoint.go:378 msg="writing series" size="11 MB"
level=debug ts=2026-02-16T18:32:49.080860548Z caller=checkpoint.go:390 msg="flushing series" totalSize="11 MB" series=2
level=info ts=2026-02-16T18:32:51.307822412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:32:51.311839224Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:32:52.929528178Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB"
level=warn ts=2026-02-16T18:32:53.071283954Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:32:56.787025827Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=info ts=2026-02-16T18:32:57.217219267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:32:57.225337788Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:32:58.071970036Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:32:58.76001288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:32:58.763948476Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:33:00.302939436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="471 B" avg_comp="471 B" total_uncomp="354 B" avg_uncomp="354 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:33:00.306648998Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:33:00.643948966Z caller=checkpoint.go:378 msg="writing series" size="681 kB"
level=warn ts=2026-02-16T18:33:03.071130827Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:33:04.501306261Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=debug ts=2026-02-16T18:33:06.199421324Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:33:06.945709967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:33:06.952363034Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T18:33:08.071400003Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:33:08.358109043Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=info ts=2026-02-16T18:33:08.745716154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:33:08.749793306Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:33:08.982439666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:33:08.986611755Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:33:10.284637529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:33:10.288626165Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-02-16T18:33:11.222536122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=debug ts=2026-02-16T18:33:12.215520395Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=info ts=2026-02-16T18:33:12.984933804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:33:12.994023223Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:33:13.053825995Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:33:13.054879618Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:33:13.071380786Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:33:13.071418196Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:33:13.071431488Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:33:13.071447974Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:33:13.744973933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:33:13.74894087Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:33:16.072341478Z caller=checkpoint.go:378 msg="writing series" size="85 kB"
level=info ts=2026-02-16T18:33:16.409036514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:33:16.41252808Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:33:16.973962967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="28 kB" avg_comp="14 kB" total_uncomp="393 kB" avg_uncomp="197 kB" idle=1 full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:33:16.977682622Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:33:17.196293719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:33:17.199776195Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:33:18.059560796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:33:18.063271472Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:33:18.071397212Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:33:19.422800119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.9 kB" avg_uncomp="4.4 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:33:19.426259087Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
ts=2026-02-16T18:33:19.450006258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:54850"
level=info ts=2026-02-16T18:33:19.55411068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:33:19.557762639Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:33:19.930128146Z caller=checkpoint.go:378 msg="writing series" size="228 kB"
level=debug ts=2026-02-16T18:33:19.930172339Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-02-16T18:33:20.492765209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:33:20.496491274Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:33:22.407201364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:33:22.410974699Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:33:23.071909624Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:33:23.786751454Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T18:33:25.243130462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:33:25.247219964Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:33:25.438700484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:33:25.442435656Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:33:27.643587581Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=warn ts=2026-02-16T18:33:28.071358631Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:33:28.713821254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:33:28.718747721Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:33:29.567586299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:33:29.570919443Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:33:30.637288829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:33:30.640774865Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:33:31.501307533Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=info ts=2026-02-16T18:33:32.82339374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:33:32.826688133Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:33:33.071069725Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:33:33.958992233Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:33:35.357743798Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-02-16T18:33:38.071721237Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:33:39.2151657Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=info ts=2026-02-16T18:33:40.06038023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:33:40.064604113Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:33:40.678380103Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:33:41.225964101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=warn ts=2026-02-16T18:33:43.071754981Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:33:43.071755473Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:33:43.071784553Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:33:43.071793655Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:33:43.071863713Z caller=checkpoint.go:390 msg="flushing series" totalSize="276 kB" series=5
level=info ts=2026-02-16T18:33:43.075527133Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-02-16T18:33:43.080603775Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.008823924s
level=info ts=2026-02-16T18:33:46.211908969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:33:46.217764343Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:33:48.07185107Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:33:50.076844825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:33:50.080792146Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:33:53.071324478Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:33:56.879119848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:33:56.883430863Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:33:57.093386348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="471 B" avg_comp="471 B" total_uncomp="354 B" avg_uncomp="354 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:33:57.09709752Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:33:58.071485181Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:34:00.110880769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:34:00.114839155Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:34:00.565204803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:34:00.56895208Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:34:01.669050919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:34:01.672982943Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:34:01.686053306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:34:01.689751301Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:34:01.870365313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:34:01.87766346Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:34:02.797580669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:34:02.80173462Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:34:03.07196534Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:34:03.503504225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:34:03.506705792Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:34:03.858780493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="28 kB" avg_comp="14 kB" total_uncomp="393 kB" avg_uncomp="197 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:34:03.862052843Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T18:34:03.862073741Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:34:03.862100685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="28 kB" avg_comp="14 kB" total_uncomp="393 kB" avg_uncomp="197 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:34:03.864762857Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:34:04.223056771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:34:04.226262712Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:34:06.429797615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:34:06.433023242Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T18:34:08.071823585Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:34:11.157204871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:34:11.161420137Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T18:34:11.228615498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:34:11.27227251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:34:11.275871293Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:34:12.706058551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:34:12.709665073Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:34:12.709686125Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:34:12.709709994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:34:12.712548201Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:34:13.053952778Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:34:13.055008282Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:34:13.071245981Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-02-16T18:34:13.071253305Z caller=checkpoint.go:612 msg="starting checkpoint"
level=debug ts=2026-02-16T18:34:13.071267759Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:34:13.07127604Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:34:13.071308226Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:34:13.071396587Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=debug ts=2026-02-16T18:34:13.074057998Z caller=checkpoint.go:378 msg="writing series" size="5.5 MB"
level=debug ts=2026-02-16T18:34:13.074069167Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.5 MB" series=1
level=info ts=2026-02-16T18:34:13.112435326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:34:13.116025134Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:34:15.153555728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:34:15.157125417Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:34:15.357769481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:34:15.361899646Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T18:34:15.714056052Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:34:16.875596935Z caller=checkpoint.go:378 msg="writing series" size="419 kB"
level=warn ts=2026-02-16T18:34:18.071392637Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:34:18.789501652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.9 kB" avg_uncomp="4.4 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:34:18.794404851Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:34:20.430911989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="28 kB" avg_comp="14 kB" total_uncomp="393 kB" avg_uncomp="197 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:34:20.434356389Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T18:34:20.678368015Z caller=checkpoint.go:378 msg="writing series" size="128 kB"
level=info ts=2026-02-16T18:34:21.048423424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:34:21.051780634Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:34:23.071931666Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:34:24.481215307Z caller=checkpoint.go:378 msg="writing series" size="82 kB"
level=info ts=2026-02-16T18:34:27.126396712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:34:27.129882855Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:34:27.410692783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:34:27.417301797Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:34:28.071287731Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:34:28.283356637Z caller=checkpoint.go:378 msg="writing series" size="128 kB"
level=info ts=2026-02-16T18:34:28.541567888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:34:28.545277837Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:34:28.840977311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:34:29.376704583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:34:29.379863281Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:34:32.086653929Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=warn ts=2026-02-16T18:34:33.071903047Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:34:35.888932969Z caller=checkpoint.go:378 msg="writing series" size="510 B"
level=warn ts=2026-02-16T18:34:38.071083411Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:34:38.942836262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="471 B" avg_comp="471 B" total_uncomp="354 B" avg_uncomp="354 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:34:38.948145067Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:34:39.692866555Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
ts=2026-02-16T18:34:41.231670661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=warn ts=2026-02-16T18:34:43.071485862Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:34:43.071493339Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:34:43.071516895Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:34:43.071526598Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:34:43.494919791Z caller=checkpoint.go:378 msg="writing series" size="917 B"
level=info ts=2026-02-16T18:34:44.099932197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:34:44.105066295Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:34:44.105092055Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:34:44.105117822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:34:44.108860369Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:34:45.256298779Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:34:47.296538767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:34:47.298899508Z caller=checkpoint.go:378 msg="writing series" size="390 kB"
level=debug ts=2026-02-16T18:34:47.298947286Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=9
level=error ts=2026-02-16T18:34:47.302084968Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T18:34:47.375415854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:35888"
level=warn ts=2026-02-16T18:34:48.071255939Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:34:49.638900909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:34:49.643258864Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:34:49.910291076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:34:49.913767313Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:34:50.775270962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:34:50.779729761Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:34:50.779759048Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:34:50.779794352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:34:50.783444921Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:34:51.100608099Z caller=checkpoint.go:378 msg="writing series" size="191 kB"
level=info ts=2026-02-16T18:34:51.323280757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:34:51.327038401Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T18:34:53.07146802Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:34:53.632724028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:34:53.63652939Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:34:53.927765548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:34:53.931877033Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T18:34:54.903709755Z caller=checkpoint.go:378 msg="writing series" size="111 kB"
level=info ts=2026-02-16T18:34:57.152376975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:34:57.159168793Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:34:57.378292385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:34:57.382144551Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T18:34:57.382181534Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:34:57.382204503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:34:57.385392349Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:34:57.91376211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:34:57.917334277Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:34:57.917360419Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=warn ts=2026-02-16T18:34:58.07187015Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:34:58.218210725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:34:58.221649241Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:34:58.428397032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:34:58.431961325Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:34:58.461142587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:34:58.464751581Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:34:58.464783164Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:34:58.518031779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:34:58.52178211Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T18:34:58.601022016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="38 kB" avg_comp="19 kB" total_uncomp="393 kB" avg_uncomp="197 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:34:58.604703586Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T18:34:58.706185203Z caller=checkpoint.go:378 msg="writing series" size="163 kB"
level=info ts=2026-02-16T18:34:59.334702905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.9 kB" avg_uncomp="4.4 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:34:59.337942868Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T18:34:59.337965775Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:34:59.337989679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.9 kB" avg_uncomp="4.4 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:34:59.341029053Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:35:00.371680016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:35:00.381048589Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:35:00.964629622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:35:00.96793567Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:35:00.967956299Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:35:00.96797837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:35:00.97164448Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:35:02.246508224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:35:02.250703173Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:35:02.50889289Z caller=checkpoint.go:378 msg="writing series" size="690 kB"
level=debug ts=2026-02-16T18:35:02.508921518Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-02-16T18:35:03.07123749Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:35:06.311754974Z caller=checkpoint.go:378 msg="writing series" size="447 kB"
ts=2026-02-16T18:35:06.459442184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:34864"
level=info ts=2026-02-16T18:35:06.891206646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:35:06.895435029Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:35:06.89546425Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:35:06.89549972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:35:06.898855007Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:35:07.106437337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:35:07.109976879Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:35:08.071816108Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:35:09.180193604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:35:09.184201997Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:35:09.933082899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:35:09.935394322Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:35:09.935416335Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:35:09.935438946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:35:09.937284783Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:35:09.939743859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:35:09.948982389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:35:09.952233764Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:35:09.952252585Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=debug ts=2026-02-16T18:35:10.114857416Z caller=checkpoint.go:378 msg="writing series" size="404 kB"
level=info ts=2026-02-16T18:35:10.235984857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:35:10.239633283Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
ts=2026-02-16T18:35:11.234547827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:35:12.349988979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:35:12.353460034Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c67a4de0d%3A19c67a4de0e%3A798d64dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:35:12.353478238Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:35:12.353500943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="43 kB" avg_comp="43 kB" total_uncomp="294 kB" avg_uncomp="294 kB" idle=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:35:12.356507348Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T18:35:13.053772065Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:35:13.054818588Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T18:35:13.066093791Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:35:13.071173549Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:35:13.071189071Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:35:13.071196718Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:35:13.071204878Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:35:13.917604205Z caller=checkpoint.go:378 msg="writing series" size="850 kB"
level=debug ts=2026-02-16T18:35:13.917635321Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=3
level=info ts=2026-02-16T18:35:15.609598849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.9 kB" avg_uncomp="4.4 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:35:15.613635284Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:35:17.51170654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:35:17.515374711Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T18:35:17.714924407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:35:17.718466068Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:35:17.720522913Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=info ts=2026-02-16T18:35:17.797819441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:35:17.801588407Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T18:35:18.071678354Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:35:19.457197844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:32936"
level=info ts=2026-02-16T18:35:20.158576339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:35:20.163493094Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:35:20.163520197Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:35:20.163551269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:35:20.16656227Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:35:21.522502363Z caller=checkpoint.go:378 msg="writing series" size="383 kB"
level=warn ts=2026-02-16T18:35:23.071740529Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:35:23.36470604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="294 kB" avg_uncomp="294 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:35:23.368419974Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:35:23.620275219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:35:23.625334525Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:35:23.62535837Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:35:23.625381787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:35:23.629629486Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:35:24.240744017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:35:24.244058338Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T18:35:25.326222524Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-02-16T18:35:28.071237484Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:35:29.129099313Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=info ts=2026-02-16T18:35:29.460018856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:35:29.464119107Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:35:29.556379565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:35:29.559905244Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:35:30.331486154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:35:30.33505158Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:35:30.33507412Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:35:30.335101609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:35:30.337966593Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:35:32.47918938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:35:32.482868453Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:35:32.931724667Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-02-16T18:35:32.931749443Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=5
level=warn ts=2026-02-16T18:35:33.071262694Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:35:35.603710839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="597 B" avg_comp="298 B" total_uncomp="694 B" avg_uncomp="347 B" idle=1 full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:35:35.609770034Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:35:35.735223393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:35:35.738690829Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:35:36.734564857Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB"
level=warn ts=2026-02-16T18:35:38.071755859Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:35:38.317753848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:35:38.321303759Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:35:38.321325984Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:35:38.321358819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:35:38.32419289Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:35:38.988583693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:35:38.992236818Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:35:39.456519864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="328 kB" avg_comp="328 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:35:39.461683576Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:35:40.536820389Z caller=checkpoint.go:378 msg="writing series" size="228 kB"
level=info ts=2026-02-16T18:35:40.591018831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:35:40.597881767Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:35:40.928753985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:35:40.933819446Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:35:40.933869772Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:35:41.237775765Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:35:41.826954482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.7 kB" avg_comp="3.7 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:35:41.830395476Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:35:42.270892429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.9 kB" avg_uncomp="4.4 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:35:42.274220703Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T18:35:43.071667639Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:35:43.071661965Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:35:43.071695466Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:35:43.071705892Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:35:43.776115589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:35:43.779784967Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T18:35:44.339916815Z caller=checkpoint.go:378 msg="writing series" size="2.5 kB"
level=info ts=2026-02-16T18:35:45.403564552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:35:45.406936252Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T18:35:46.397737913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:35:46.401566284Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:35:46.676487315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:35:46.680125814Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:35:47.717815135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:35:47.721524067Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:35:47.721545258Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:35:47.721570125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:35:47.724361129Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:35:48.071400606Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:35:48.142905976Z caller=checkpoint.go:378 msg="writing series" size="707 kB"
level=info ts=2026-02-16T18:35:48.728525737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:35:48.731917135Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:35:49.447706158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:35:49.450887637Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T18:35:49.459750685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:40636"
level=info ts=2026-02-16T18:35:49.941102254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:35:49.944386799Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:35:50.920527273Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:35:51.9458056Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-02-16T18:35:53.071268876Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:35:53.078371731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="294 kB" avg_uncomp="294 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:35:53.081953078Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T18:35:55.748028501Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T18:35:56.291701496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:35:56.295072496Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:35:56.410257052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:35:56.413735778Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:35:57.258506168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="38 kB" avg_comp="19 kB" total_uncomp="393 kB" avg_uncomp="197 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:35:57.2619027Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:35:57.917558085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:35:57.921413321Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:35:58.071959579Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:35:58.928450948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:35:58.9321757Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:35:59.550818728Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=debug ts=2026-02-16T18:35:59.550851695Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-02-16T18:36:01.973205707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:36:01.976589397Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T18:36:03.071762531Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:36:03.353887193Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=info ts=2026-02-16T18:36:03.365949126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:36:03.370016352Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:36:06.091871967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="917 B" avg_comp="917 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:36:06.095972679Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T18:36:06.463034695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:59784"
level=debug ts=2026-02-16T18:36:07.156536691Z caller=checkpoint.go:378 msg="writing series" size="216 kB"
level=warn ts=2026-02-16T18:36:08.071978812Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:36:10.95948281Z caller=checkpoint.go:378 msg="writing series" size="239 kB"
ts=2026-02-16T18:36:11.241131656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:36:11.72798086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:36:11.732328597Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:36:13.053262475Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:36:13.054336837Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:36:13.071619904Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:36:13.071666202Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T18:36:13.071657216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:36:13.071676772Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:36:14.552666866Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:36:14.762406978Z caller=checkpoint.go:378 msg="writing series" size="4.5 kB"
level=info ts=2026-02-16T18:36:16.936443944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:36:16.943110871Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T18:36:18.071864547Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:36:18.321625068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="597 B" avg_comp="298 B" total_uncomp="694 B" avg_uncomp="347 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:36:18.325050015Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:36:18.564654721Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T18:36:19.439023538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:36:19.444117058Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-02-16T18:36:19.462209371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:39854"
level=info ts=2026-02-16T18:36:19.771986986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="432 kB" avg_comp="216 kB" total_uncomp="4.6 MB" avg_uncomp="2.3 MB" idle=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:36:19.776240101Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:36:22.36793594Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=warn ts=2026-02-16T18:36:23.071109078Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:36:24.891443283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:36:24.895896503Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T18:36:26.170387053Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-02-16T18:36:26.986708784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.2 kB" avg_comp="1.6 kB" total_uncomp="8.9 kB" avg_uncomp="4.4 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:36:26.990676423Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:36:27.944458468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:36:27.948078456Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T18:36:28.071684329Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:36:28.371609422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:36:28.37508385Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:36:29.50683201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:36:29.510968056Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:36:29.973605354Z caller=checkpoint.go:378 msg="writing series" size="263 kB"
level=info ts=2026-02-16T18:36:30.522188876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:36:30.526047724Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:36:30.881267906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:36:30.885175389Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:36:31.071325899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:36:31.077571457Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:36:33.071942713Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:36:33.77691412Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T18:36:34.945295263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:36:34.94906885Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:36:35.347929238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:36:35.349821374Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:36:36.466515932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58776"
level=debug ts=2026-02-16T18:36:37.579183249Z caller=checkpoint.go:378 msg="writing series" size="81 kB"
level=info ts=2026-02-16T18:36:37.793875596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:36:37.799057392Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-02-16T18:36:38.071882389Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:36:38.524055968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:36:38.527595911Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:36:40.530420963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:36:40.534125985Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-02-16T18:36:41.24424618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=debug ts=2026-02-16T18:36:41.382013033Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T18:36:43.071363244Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:36:43.07139053Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:36:43.071400965Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:36:43.071413309Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:36:45.185126317Z caller=checkpoint.go:378 msg="writing series" size="8.3 kB"
level=info ts=2026-02-16T18:36:45.638539475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="38 kB" avg_comp="19 kB" total_uncomp="393 kB" avg_uncomp="197 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:36:45.64388334Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T18:36:47.144114666Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:36:47.382706954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:48766"
level=info ts=2026-02-16T18:36:47.784983703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:36:47.788830323Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:36:47.825930643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:36:47.829290853Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:36:48.071220732Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:36:48.98733034Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
ts=2026-02-16T18:36:49.46503056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:60468"
level=info ts=2026-02-16T18:36:49.944196251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:36:49.947431467Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T18:36:52.790721837Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=debug ts=2026-02-16T18:36:52.790749203Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=14
level=info ts=2026-02-16T18:36:52.846044711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="294 kB" avg_uncomp="294 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:36:52.84977492Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T18:36:53.071625251Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:36:56.593788312Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
level=info ts=2026-02-16T18:36:57.89159129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:36:57.895464926Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:36:58.07116581Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:36:58.569391026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:36:58.573592052Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:36:59.668194926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:36:59.671721229Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:37:00.395617685Z caller=checkpoint.go:378 msg="writing series" size="900 B"
level=warn ts=2026-02-16T18:37:03.070976409Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:37:04.199348261Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T18:37:07.071541347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:37:07.077029678Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:37:08.001422232Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T18:37:08.071340469Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:37:09.291361561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="432 kB" avg_comp="216 kB" total_uncomp="4.6 MB" avg_uncomp="2.3 MB" full=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:37:09.294948159Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:37:10.13727435Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:37:10.502336936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:37:10.50582242Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:37:10.722444605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:37:10.725741509Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:37:10.802975579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:37:10.806518439Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-02-16T18:37:11.247261135Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:37:11.661803586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:37:11.664984731Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:37:11.804293285Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=debug ts=2026-02-16T18:37:13.053755961Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:37:13.054796917Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:37:13.071239024Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:37:13.071274204Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:37:13.071284985Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:37:13.07128268Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:37:14.071244631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="579 B" avg_comp="289 B" total_uncomp="694 B" avg_uncomp="347 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:37:14.074834583Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:37:15.607810471Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=warn ts=2026-02-16T18:37:18.071197498Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:37:19.410261272Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
ts=2026-02-16T18:37:19.467327395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:51830"
level=info ts=2026-02-16T18:37:20.806652282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:37:20.814482443Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:37:21.28290356Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:37:21.75928271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:37:21.762610018Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-02-16T18:37:23.071211767Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:37:23.212526778Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=info ts=2026-02-16T18:37:23.372386366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:37:23.376294538Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:37:23.376317045Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:37:23.37634082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:37:23.379280735Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:37:25.820752626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.3 kB" avg_comp="1.6 kB" total_uncomp="8.9 kB" avg_uncomp="4.4 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:37:25.824784956Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:37:25.838002803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:37:25.841794875Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:37:27.016216106Z caller=checkpoint.go:378 msg="writing series" size="231 kB"
level=warn ts=2026-02-16T18:37:28.071656266Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:37:28.105728337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:37:28.109562451Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:37:30.81948458Z caller=checkpoint.go:378 msg="writing series" size="706 kB"
level=debug ts=2026-02-16T18:37:30.81951576Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=10
level=warn ts=2026-02-16T18:37:33.071822954Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:37:33.117983008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:37:33.121646185Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:37:33.732079438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:37:33.736017382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:37:33.736125753Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:37:33.736144874Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:37:33.736187981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:37:33.739266945Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:37:33.739275571Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:37:34.621630897Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=info ts=2026-02-16T18:37:35.244649649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:37:35.24813053Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:37:35.270330902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="38 kB" avg_comp="19 kB" total_uncomp="393 kB" avg_uncomp="197 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:37:35.273553746Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:37:36.749763362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:37:36.753720764Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:37:38.071341985Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:37:38.424744758Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T18:37:38.945856096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:37:38.951206711Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:37:40.147357385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:37:40.151868734Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
ts=2026-02-16T18:37:41.250040728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=debug ts=2026-02-16T18:37:42.228231383Z caller=checkpoint.go:378 msg="writing series" size="2.1 MB"
level=debug ts=2026-02-16T18:37:42.228255481Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.1 MB" series=3
level=info ts=2026-02-16T18:37:43.071164143Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:37:43.071193553Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:37:43.071201659Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:37:43.071199812Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:37:43.805205601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="294 kB" avg_uncomp="294 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:37:43.808992658Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:37:44.72338224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="587 B" avg_comp="587 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:37:44.727411589Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:37:45.594621379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:37:45.598671076Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:37:46.030033872Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
level=debug ts=2026-02-16T18:37:46.162671806Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:37:46.226740374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:37:46.23077545Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:37:48.071818343Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:37:49.351301243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:37:49.355112686Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T18:37:49.470491879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:39116"
level=debug ts=2026-02-16T18:37:49.851741615Z caller=checkpoint.go:378 msg="writing series" size="17 MB"
level=debug ts=2026-02-16T18:37:49.851768686Z caller=checkpoint.go:390 msg="flushing series" totalSize="17 MB" series=2
level=info ts=2026-02-16T18:37:51.370744526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:37:51.381181742Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:37:53.071611706Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:37:53.635951195Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB"
level=info ts=2026-02-16T18:37:54.686828582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" idle=1 full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:37:54.690301076Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:37:57.438237048Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=warn ts=2026-02-16T18:37:58.072058892Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:37:58.816299055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:37:58.820710621Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:37:58.820744474Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:37:58.820777563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:37:58.82441993Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:37:59.242790908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:37:59.246298975Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:37:59.261710602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:38:00.122206034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="597 B" avg_comp="298 B" total_uncomp="694 B" avg_uncomp="347 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:38:00.125889676Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:38:01.057785318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="432 kB" avg_comp="216 kB" total_uncomp="4.6 MB" avg_uncomp="2.3 MB" full=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:38:01.061606525Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:38:01.241298415Z caller=checkpoint.go:378 msg="writing series" size="763 kB"
level=warn ts=2026-02-16T18:38:03.071736873Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:38:03.29777761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:38:03.301328669Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T18:38:03.955651159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:38:03.966690111Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:38:03.966720482Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:38:03.966754109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:38:03.972816553Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:38:05.043793618Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T18:38:06.674076456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:38:06.677664015Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T18:38:08.071548329Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:38:08.814731941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:38:08.818785497Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:38:08.847200437Z caller=checkpoint.go:378 msg="writing series" size="270 kB"
level=debug ts=2026-02-16T18:38:08.847223024Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-02-16T18:38:09.025572888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:38:09.031409999Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:38:09.945122733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:38:09.948609364Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T18:38:11.252702313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=debug ts=2026-02-16T18:38:11.362474539Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:38:11.368516763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.3 kB" avg_comp="1.6 kB" total_uncomp="8.9 kB" avg_uncomp="4.4 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:38:11.371856903Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:38:12.276426538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:38:12.279927538Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:38:12.649950474Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=debug ts=2026-02-16T18:38:13.053505587Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:38:13.054568808Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:38:13.071854731Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:38:13.071877368Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:38:13.071884693Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:38:13.071896111Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:38:16.452508267Z caller=checkpoint.go:378 msg="writing series" size="98 kB"
level=warn ts=2026-02-16T18:38:18.071261003Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:38:18.600204976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="38 kB" avg_comp="19 kB" total_uncomp="393 kB" avg_uncomp="197 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:38:18.604400604Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:38:18.923494109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:38:18.927618848Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
ts=2026-02-16T18:38:19.473522156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:34680"
level=info ts=2026-02-16T18:38:20.136345112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:38:20.143004096Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:38:20.25574839Z caller=checkpoint.go:378 msg="writing series" size="469 kB"
level=warn ts=2026-02-16T18:38:23.071416715Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:38:23.196656476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:38:23.20250275Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:38:24.058352524Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T18:38:25.354610305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:38:25.358604921Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:38:27.693508443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:38:27.697184821Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:38:27.860892889Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=warn ts=2026-02-16T18:38:28.071493853Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:38:31.117441279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="294 kB" avg_uncomp="294 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:38:31.121082269Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T18:38:31.663853794Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=info ts=2026-02-16T18:38:32.341775063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:38:32.345187206Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:38:32.396256197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:38:32.399380959Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:38:32.399403045Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:38:32.399426856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:38:32.401976598Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:38:32.928122937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:38:32.93158739Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T18:38:33.071212074Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:38:34.660772917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:38:34.663948362Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:38:35.466422642Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T18:38:37.321677155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:38:37.327830157Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:38:37.429683606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:38:37.433535808Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:38:38.07170442Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:38:39.269389359Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
ts=2026-02-16T18:38:41.255633805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:38:43.071170433Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:38:43.07120083Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:38:43.071209779Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:38:43.071222966Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:38:43.072285096Z caller=checkpoint.go:390 msg="flushing series" totalSize="876 kB" series=8
level=info ts=2026-02-16T18:38:43.075378301Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-02-16T18:38:43.07772223Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.006121438s
level=info ts=2026-02-16T18:38:43.331010194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:38:43.336063389Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:38:44.119278137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:38:44.124580539Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-02-16T18:38:44.125755096Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:38:44.21737734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:38:44.221028976Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:38:46.556036675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="715 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:38:46.559886188Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T18:38:47.390321366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:50782"
level=warn ts=2026-02-16T18:38:48.071205206Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:38:51.62870669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:38:51.632473557Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:38:51.857369031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:38:51.860907975Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:38:52.492694184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="454 kB" avg_comp="227 kB" total_uncomp="4.6 MB" avg_uncomp="2.3 MB" full=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:38:52.49695855Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:38:53.071723577Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:38:54.047630894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.3 kB" avg_comp="1.6 kB" total_uncomp="8.9 kB" avg_uncomp="4.4 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:38:54.052836437Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:38:56.340426604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="597 B" avg_comp="298 B" total_uncomp="694 B" avg_uncomp="347 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:38:56.343673322Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:38:56.343696554Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:38:56.343718384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="597 B" avg_comp="298 B" total_uncomp="694 B" avg_uncomp="347 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:38:56.34679845Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:38:58.071628779Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:38:59.610564365Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:39:00.290065575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:39:00.293422568Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-02-16T18:39:03.071454264Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:39:04.597931975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:39:04.601637706Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:39:04.877691387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="38 kB" avg_comp="19 kB" total_uncomp="393 kB" avg_uncomp="197 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:39:04.88132271Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:39:05.419727354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:39:05.423243921Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:39:07.302810524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:39:07.306221852Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:39:07.938084507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:39:07.941792563Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:39:08.071269213Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:39:11.258001596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T18:39:12.967831691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:39:12.971742183Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:39:13.053687377Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:39:13.054752481Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:39:13.070960928Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-02-16T18:39:13.071032281Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:39:13.071033915Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:39:13.071054051Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:39:13.07106321Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:39:13.071168021Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=debug ts=2026-02-16T18:39:13.074612501Z caller=checkpoint.go:378 msg="writing series" size="6.4 MB"
level=debug ts=2026-02-16T18:39:13.074624307Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.4 MB" series=1
level=info ts=2026-02-16T18:39:13.423269757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="597 B" avg_comp="298 B" total_uncomp="694 B" avg_uncomp="347 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:39:13.427945914Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:39:13.79790742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:39:13.801627591Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:39:14.681983947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="294 kB" avg_uncomp="294 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:39:14.685414276Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T18:39:16.874874022Z caller=checkpoint.go:378 msg="writing series" size="307 kB"
level=info ts=2026-02-16T18:39:17.183668148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:39:17.187263375Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:39:17.59773644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:39:17.601015597Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:39:18.023228094Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:39:18.023272581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4c0d02f8eff28ba8 immediate=false num_chunks=1 total_comp="104 kB" avg_comp="104 kB" total_uncomp="673 kB" avg_uncomp="673 kB" idle=1 labels="{stream=\"gitlab-rep-qtrv\"}"
level=error ts=2026-02-16T18:39:18.028793518Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4c0d02f8eff28ba8/19c67b2c8ae%3A19c67bb3712%3A238c6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qtrv\"}"
level=warn ts=2026-02-16T18:39:18.071187368Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:39:18.323977564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="217 kB" avg_comp="217 kB" total_uncomp="833 kB" avg_uncomp="833 kB" idle=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:39:18.327502096Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T18:39:20.635179984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:39:20.638825107Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:39:20.677229315Z caller=checkpoint.go:378 msg="writing series" size="128 kB"
level=info ts=2026-02-16T18:39:22.210792802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:39:22.215348294Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:39:23.071118978Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:39:24.480430264Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=info ts=2026-02-16T18:39:27.356629136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:39:27.361179596Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T18:39:28.071259398Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:39:28.215557341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4c0d02f8eff28ba8 immediate=false num_chunks=1 total_comp="139 kB" avg_comp="139 kB" total_uncomp="673 kB" avg_uncomp="673 kB" full=1 labels="{stream=\"gitlab-rep-qtrv\"}"
level=error ts=2026-02-16T18:39:28.219210126Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4c0d02f8eff28ba8/19c67b2c8ae%3A19c67bb3712%3A238c6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qtrv\"}"
level=debug ts=2026-02-16T18:39:28.282731755Z caller=checkpoint.go:378 msg="writing series" size="163 kB"
level=info ts=2026-02-16T18:39:28.996336682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:39:29.000188381Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T18:39:29.991370348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:39:29.994664899Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:39:32.085750217Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=info ts=2026-02-16T18:39:32.194059644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:39:32.197501438Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T18:39:33.071275488Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:39:35.124385353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:39:35.127998825Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:39:35.889127544Z caller=checkpoint.go:378 msg="writing series" size="510 B"
level=info ts=2026-02-16T18:39:36.359616329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:39:36.363210995Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:39:36.511674681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.3 kB" avg_comp="1.6 kB" total_uncomp="8.9 kB" avg_uncomp="4.4 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:39:36.515062631Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T18:39:38.071946196Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:39:39.691414333Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=info ts=2026-02-16T18:39:40.235278565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:39:40.238729045Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T18:39:41.260561608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=debug ts=2026-02-16T18:39:41.776550123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:39:42.322452364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:39:42.328782525Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:39:43.071033712Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:39:43.071071347Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:39:43.071083444Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:39:43.071073233Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:39:43.494828363Z caller=checkpoint.go:378 msg="writing series" size="917 B"
level=debug ts=2026-02-16T18:39:45.082610811Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:39:45.390279074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:39:45.395102738Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:39:46.516282818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="597 B" avg_comp="298 B" total_uncomp="694 B" avg_uncomp="347 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:39:46.519639446Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:39:47.29699277Z caller=checkpoint.go:378 msg="writing series" size="305 kB"
level=debug ts=2026-02-16T18:39:47.297018216Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
ts=2026-02-16T18:39:47.39431714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:56380"
level=warn ts=2026-02-16T18:39:48.071306304Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:39:51.100617835Z caller=checkpoint.go:378 msg="writing series" size="261 kB"
level=info ts=2026-02-16T18:39:51.873018531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:39:51.87700742Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:39:51.903100534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="454 kB" avg_comp="227 kB" total_uncomp="4.6 MB" avg_uncomp="2.3 MB" full=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:39:51.906752944Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:39:53.070997411Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:39:53.433136088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:39:53.436568052Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:39:54.794859551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="38 kB" avg_comp="19 kB" total_uncomp="393 kB" avg_uncomp="197 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:39:54.798807175Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T18:39:54.903276488Z caller=checkpoint.go:378 msg="writing series" size="152 kB"
level=info ts=2026-02-16T18:39:56.012304031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:39:56.015650135Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-02-16T18:39:58.071703606Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:39:58.706020817Z caller=checkpoint.go:378 msg="writing series" size="201 kB"
level=info ts=2026-02-16T18:40:00.82942591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:40:00.835282007Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:40:01.197452918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:40:01.200992996Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:40:02.482743754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4c0d02f8eff28ba8 immediate=false num_chunks=1 total_comp="139 kB" avg_comp="139 kB" total_uncomp="673 kB" avg_uncomp="673 kB" full=1 labels="{stream=\"gitlab-rep-qtrv\"}"
level=error ts=2026-02-16T18:40:02.486549698Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4c0d02f8eff28ba8/19c67b2c8ae%3A19c67bb3712%3A238c6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qtrv\"}"
level=debug ts=2026-02-16T18:40:02.50900161Z caller=checkpoint.go:378 msg="writing series" size="759 kB"
level=debug ts=2026-02-16T18:40:02.509028217Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-02-16T18:40:03.071267775Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:40:04.671726774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="294 kB" avg_uncomp="294 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:40:04.67524332Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:40:05.379209978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:40:05.382570516Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:40:06.081620413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:40:06.086556828Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:40:06.311358084Z caller=checkpoint.go:378 msg="writing series" size="442 kB"
ts=2026-02-16T18:40:06.476218412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58310"
level=info ts=2026-02-16T18:40:07.223275989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:40:07.227000544Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:40:08.071666618Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:40:08.546896508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:40:08.550545907Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T18:40:08.843007399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:40:08.847126394Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:40:10.114639717Z caller=checkpoint.go:378 msg="writing series" size="605 kB"
level=info ts=2026-02-16T18:40:10.489604341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:40:10.493219748Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
ts=2026-02-16T18:40:11.263604983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:40:12.010995947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:40:12.014891188Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:40:12.391002178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:40:12.394599247Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:40:13.053996514Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:40:13.05503607Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:40:13.071348732Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:40:13.071376213Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:40:13.071384261Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:40:13.071416425Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:40:13.918280897Z caller=checkpoint.go:378 msg="writing series" size="889 kB"
level=debug ts=2026-02-16T18:40:13.918306373Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=3
level=info ts=2026-02-16T18:40:13.953454165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:40:13.956817303Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:40:14.045789697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:40:14.049005314Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:40:15.815142952Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:40:17.719788909Z caller=checkpoint.go:378 msg="writing series" size="5.6 kB"
level=warn ts=2026-02-16T18:40:18.071946162Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:40:19.480140212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:47838"
level=debug ts=2026-02-16T18:40:21.522727062Z caller=checkpoint.go:378 msg="writing series" size="280 kB"
level=info ts=2026-02-16T18:40:22.404936314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:40:22.409637184Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:40:23.070965076Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:40:25.325315719Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T18:40:25.619201275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:40:25.623069521Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:40:26.624693983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:40:26.628659676Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:40:26.628683918Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:40:26.628706258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:40:26.631602156Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:40:28.021961445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:40:28.02622729Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T18:40:28.071653422Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:40:29.128431008Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
level=info ts=2026-02-16T18:40:30.632763986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.3 kB" avg_comp="1.6 kB" total_uncomp="8.9 kB" avg_uncomp="4.4 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:40:30.636926858Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:40:32.354913879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="454 kB" avg_comp="227 kB" total_uncomp="4.6 MB" avg_uncomp="2.3 MB" full=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:40:32.362435578Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:40:32.93260112Z caller=checkpoint.go:378 msg="writing series" size="2.2 MB"
level=debug ts=2026-02-16T18:40:32.932628479Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.5 MB" series=5
level=warn ts=2026-02-16T18:40:33.071073416Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:40:33.537780007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:40:33.541575069Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:40:36.734090338Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB"
level=info ts=2026-02-16T18:40:37.191446345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:40:37.198585473Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:40:37.803086218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:40:37.806657112Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-02-16T18:40:38.0717448Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:40:39.889754098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:40:39.891518968Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T18:40:40.385029545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="597 B" avg_comp="298 B" total_uncomp="694 B" avg_uncomp="347 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:40:40.388650106Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:40:40.53728312Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
ts=2026-02-16T18:40:41.266379485Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T18:40:42.527770974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:40:42.531338181Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:40:43.071123544Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:40:43.071167854Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:40:43.071178976Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:40:43.071190472Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:40:44.339786412Z caller=checkpoint.go:378 msg="writing series" size="2.8 kB"
level=info ts=2026-02-16T18:40:45.10535114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:40:45.109139475Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-02-16T18:40:47.398324384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:57208"
level=warn ts=2026-02-16T18:40:48.071274802Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:40:48.142673211Z caller=checkpoint.go:378 msg="writing series" size="775 kB"
level=debug ts=2026-02-16T18:40:49.158084487Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:40:49.834676347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:40:49.838567817Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:40:50.358269917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:40:50.363708128Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T18:40:50.641346203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="38 kB" avg_comp="19 kB" total_uncomp="393 kB" avg_uncomp="197 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:40:50.645106408Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c67a4d697%3A19c67a4d698%3A23250aa3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T18:40:50.64512901Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:40:50.645169102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="124 B" avg_comp="124 B" total_uncomp="47 kB" avg_uncomp="47 kB" idle=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:40:50.647876412Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:40:51.945383226Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-02-16T18:40:53.07179563Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:40:54.015399543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:40:54.02043149Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:40:54.047649959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.7 kB" avg_comp="3.7 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:40:54.05090093Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:40:54.540268826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4c0d02f8eff28ba8 immediate=false num_chunks=1 total_comp="139 kB" avg_comp="139 kB" total_uncomp="673 kB" avg_uncomp="673 kB" full=1 labels="{stream=\"gitlab-rep-qtrv\"}"
level=error ts=2026-02-16T18:40:54.544226804Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4c0d02f8eff28ba8/19c67b2c8ae%3A19c67bb3712%3A238c6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qtrv\"}"
level=debug ts=2026-02-16T18:40:55.748032433Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T18:40:56.230277973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:40:56.233924059Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:40:56.874803491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:40:56.878440935Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:40:58.07187689Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:40:59.551003647Z caller=checkpoint.go:378 msg="writing series" size="122 kB"
level=debug ts=2026-02-16T18:40:59.551031479Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-02-16T18:41:00.795985715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="294 kB" avg_uncomp="294 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:41:00.799818607Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:41:02.42839372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:41:02.433746417Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:41:02.693649508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:41:02.697565762Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:41:03.070976836Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:41:03.353222697Z caller=checkpoint.go:378 msg="writing series" size="30 kB"
level=info ts=2026-02-16T18:41:05.336925042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:41:05.342614468Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:41:06.314772461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:41:06.31862364Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T18:41:06.481990978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53516"
level=info ts=2026-02-16T18:41:07.119779122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:41:07.123748713Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T18:41:07.156033271Z caller=checkpoint.go:378 msg="writing series" size="236 kB"
level=warn ts=2026-02-16T18:41:08.071696076Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:41:08.090761078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:41:08.094681469Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:41:08.094702359Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:41:08.094728028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:41:08.098044009Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:41:08.2143482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:41:08.218184275Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:41:08.525905918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:41:08.529681127Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:41:08.773487985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:41:08.77730027Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:41:09.134382127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:41:09.138059886Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:41:10.959226591Z caller=checkpoint.go:378 msg="writing series" size="228 kB"
ts=2026-02-16T18:41:11.269214953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=debug ts=2026-02-16T18:41:13.053651696Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:41:13.054730212Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T18:41:13.070990904Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:41:13.070998862Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:41:13.071016593Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:41:13.07102413Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:41:14.761774797Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=info ts=2026-02-16T18:41:15.561434264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="454 kB" avg_comp="227 kB" total_uncomp="4.6 MB" avg_uncomp="2.3 MB" full=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:41:15.565628851Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:41:15.565650007Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:41:15.565675268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="454 kB" avg_comp="227 kB" total_uncomp="4.6 MB" avg_uncomp="2.3 MB" full=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:41:15.56927178Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:41:16.39100105Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:41:18.071783761Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:41:18.56429183Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
ts=2026-02-16T18:41:19.483911182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:41508"
level=info ts=2026-02-16T18:41:21.210384276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:41:21.214202778Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:41:22.055694974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:41:22.059216535Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:41:22.059239694Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:41:22.059260999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:41:22.06218845Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:41:22.368182849Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=info ts=2026-02-16T18:41:22.855649991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:41:22.858928697Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:41:22.998336134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="917 B" avg_comp="917 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:41:23.001635393Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:41:23.071018024Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:41:23.307027924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="892 B" avg_comp="446 B" total_uncomp="694 B" avg_uncomp="347 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:41:23.310389035Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:41:23.442314752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:41:25.232426867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:41:25.235974275Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T18:41:25.235995822Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:41:25.23602398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:41:25.238775123Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T18:41:26.170599103Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=info ts=2026-02-16T18:41:27.872887065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:41:27.880257256Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:41:28.07162355Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:41:29.973977186Z caller=checkpoint.go:378 msg="writing series" size="401 kB"
level=info ts=2026-02-16T18:41:30.063376979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="454 kB" avg_comp="227 kB" total_uncomp="4.6 MB" avg_uncomp="2.3 MB" full=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:41:30.067425547Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:41:30.094500836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.2 kB" avg_comp="1.6 kB" total_uncomp="8.9 kB" avg_uncomp="4.4 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:41:30.09802836Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T18:41:30.09805436Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:41:30.098085612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.3 kB" avg_comp="1.6 kB" total_uncomp="8.9 kB" avg_uncomp="4.4 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:41:30.101009186Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:41:31.612287993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:41:31.615676967Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:41:31.61569844Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:41:31.615721094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:41:31.618622074Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:41:33.071209813Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:41:33.775946651Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T18:41:35.967004778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:41:36.035561718Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:41:37.579143112Z caller=checkpoint.go:378 msg="writing series" size="90 kB"
level=warn ts=2026-02-16T18:41:38.071921468Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:41:38.514985535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:41:38.518554708Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:41:40.436929166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:41:40.440273551Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:41:40.440293355Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:41:40.44031408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:41:40.44304891Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:41:40.793241975Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:41:40.793308415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="766 kB" avg_comp="766 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:41:40.797395381Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-02-16T18:41:41.271806161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=debug ts=2026-02-16T18:41:41.381411771Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T18:41:42.727083695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:41:42.730744288Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:41:43.071249494Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:41:43.071281693Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:41:43.07129057Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:41:43.071346784Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:41:45.185011414Z caller=checkpoint.go:378 msg="writing series" size="8.8 kB"
level=info ts=2026-02-16T18:41:45.614286269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:41:45.617971442Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:41:46.658050362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:41:46.662784659Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:41:47.003670086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:41:47.007358634Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T18:41:47.356521887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:41:47.359979466Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
ts=2026-02-16T18:41:47.402533343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:44612"
level=warn ts=2026-02-16T18:41:48.071951849Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:41:48.21527816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.4 kB" avg_comp="1.1 kB" total_uncomp="9.5 kB" avg_uncomp="3.2 kB" idle=1 full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:41:48.219187459Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T18:41:48.987368777Z caller=checkpoint.go:378 msg="writing series" size="143 kB"
level=debug ts=2026-02-16T18:41:48.987397773Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=13
level=info ts=2026-02-16T18:41:49.337103299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="294 kB" avg_uncomp="294 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:41:49.340716036Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:41:49.340742934Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:41:49.340777017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="294 kB" avg_uncomp="294 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:41:49.343810158Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:41:50.878316398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:41:50.883113078Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:41:50.883135003Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:41:50.883180503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:41:50.887640606Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:41:50.95091945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:41:50.954286992Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:41:51.2178942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:41:51.22343389Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T18:41:52.282390942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:41:52.286312826Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:41:52.672414905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4c0d02f8eff28ba8 immediate=false num_chunks=1 total_comp="139 kB" avg_comp="139 kB" total_uncomp="673 kB" avg_uncomp="673 kB" full=1 labels="{stream=\"gitlab-rep-qtrv\"}"
level=error ts=2026-02-16T18:41:52.67598035Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4c0d02f8eff28ba8/19c67b2c8ae%3A19c67bb3712%3A238c6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qtrv\"}"
level=debug ts=2026-02-16T18:41:52.790249701Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=warn ts=2026-02-16T18:41:53.071233934Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:41:54.617330716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:41:54.620998832Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:41:54.621020871Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:41:54.621044535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:41:54.623951761Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:41:54.92809706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:41:54.93214169Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:41:56.593081543Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=info ts=2026-02-16T18:41:57.254374306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:41:57.258215576Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:41:58.071272456Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:41:59.081738548Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:42:00.145530904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:42:00.149866875Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:42:00.335281688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:42:00.338997102Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:42:00.396223743Z caller=checkpoint.go:378 msg="writing series" size="900 B"
level=info ts=2026-02-16T18:42:00.539212366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:42:00.543333263Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:42:00.543360664Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:42:00.543391088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:42:00.546529183Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:42:02.139203373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:42:02.142656324Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:42:02.142679343Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:42:02.264216819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="294 kB" avg_uncomp="294 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:42:02.268213649Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:42:02.443453188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:42:02.447058636Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T18:42:03.07167529Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:42:04.198471308Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T18:42:04.20262925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="454 kB" avg_comp="227 kB" total_uncomp="4.6 MB" avg_uncomp="2.3 MB" full=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:42:04.207448595Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:42:06.469046826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:42:06.474554217Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:42:06.474582039Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:42:06.474614835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:42:06.477811185Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
ts=2026-02-16T18:42:06.486372522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:52144"
level=info ts=2026-02-16T18:42:07.824106433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:42:07.827830619Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:42:08.001523871Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T18:42:08.071167087Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:42:08.484427427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:42:08.488420102Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:42:08.488448037Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:42:08.488481697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:42:08.491550366Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:42:08.871922623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:42:08.87562266Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:42:09.037984345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:42:09.042989848Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-02-16T18:42:11.27474011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=debug ts=2026-02-16T18:42:11.804477072Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=debug ts=2026-02-16T18:42:13.054286216Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:42:13.054300451Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:42:13.071853346Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:42:13.071885408Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:42:13.071893544Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:42:13.071892086Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:42:14.130602546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:42:14.134197459Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:42:15.610303342Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=debug ts=2026-02-16T18:42:15.768194714Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:42:17.780960989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:42:17.784983019Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T18:42:18.071170158Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:42:18.325246349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:42:18.330109666Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T18:42:19.410290093Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=info ts=2026-02-16T18:42:19.453586064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:42:19.457254182Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
ts=2026-02-16T18:42:19.48789101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:41160"
level=info ts=2026-02-16T18:42:19.509239714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:42:19.531482629Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:42:19.5315256Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:42:19.531568973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:42:19.535075435Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:42:19.587593657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="910 B" avg_comp="455 B" total_uncomp="694 B" avg_uncomp="347 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:42:19.59108042Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:42:21.159642635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:42:21.164005099Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T18:42:21.581197095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:42:21.584570864Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:42:22.196380523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.4 kB" avg_comp="1.1 kB" total_uncomp="9.5 kB" avg_uncomp="3.2 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:42:22.199770839Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T18:42:23.071698839Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:42:23.213262829Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=info ts=2026-02-16T18:42:23.533834722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:42:23.540382829Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:42:26.694810684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:42:26.6986546Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:42:27.018054015Z caller=checkpoint.go:378 msg="writing series" size="231 kB"
level=warn ts=2026-02-16T18:42:28.071605465Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:42:29.023233666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:42:29.02850158Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T18:42:29.771654639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:42:29.776288369Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:42:30.818893945Z caller=checkpoint.go:378 msg="writing series" size="814 kB"
level=debug ts=2026-02-16T18:42:30.818933849Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=11
level=info ts=2026-02-16T18:42:31.234207814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:42:31.237810317Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:42:31.966422993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="587 B" avg_comp="587 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:42:31.970410374Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:42:32.421231879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:42:32.429542627Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:42:33.071925187Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:42:34.621277847Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=info ts=2026-02-16T18:42:34.75711711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:42:34.762038368Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:42:37.313227805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:42:37.330624663Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:42:38.019757651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="294 kB" avg_uncomp="294 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:42:38.029038384Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T18:42:38.071613901Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:42:38.425479661Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T18:42:39.116247281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4c0d02f8eff28ba8 immediate=false num_chunks=1 total_comp="139 kB" avg_comp="139 kB" total_uncomp="673 kB" avg_uncomp="673 kB" full=1 labels="{stream=\"gitlab-rep-qtrv\"}"
level=error ts=2026-02-16T18:42:39.127675087Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4c0d02f8eff28ba8/19c67b2c8ae%3A19c67bb3712%3A238c6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qtrv\"}"
level=info ts=2026-02-16T18:42:39.218232875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:42:39.228640247Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
ts=2026-02-16T18:42:41.277692893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=debug ts=2026-02-16T18:42:42.228636332Z caller=checkpoint.go:378 msg="writing series" size="2.1 MB"
level=debug ts=2026-02-16T18:42:42.228669823Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.1 MB" series=3
level=info ts=2026-02-16T18:42:43.071268132Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:42:43.071300379Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:42:43.07130953Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:42:43.071339715Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:42:44.119286713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:42:44.129587706Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:42:44.229223069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:42:44.232971252Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:42:46.029695332Z caller=checkpoint.go:378 msg="writing series" size="158 kB"
level=info ts=2026-02-16T18:42:47.667342069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:42:47.671084816Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T18:42:48.071430914Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:42:48.133529007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:42:48.137087058Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:42:49.557041763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:42:49.560413631Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:42:49.847292152Z caller=checkpoint.go:378 msg="writing series" size="20 MB"
level=debug ts=2026-02-16T18:42:49.847324115Z caller=checkpoint.go:390 msg="flushing series" totalSize="20 MB" series=2
level=debug ts=2026-02-16T18:42:50.678902235Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:42:50.934734333Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:42:50.937826001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:42:50.941128955Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:42:50.941145824Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:42:50.941181477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:42:50.943821496Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:42:52.786870967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:42:52.790162291Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:42:53.071312393Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:42:53.336570358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:42:53.33995841Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:42:53.39814496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:42:53.407857443Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:42:53.634773838Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=info ts=2026-02-16T18:42:56.59547897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:42:56.599198139Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:42:57.437876107Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-02-16T18:42:58.07120712Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:42:58.964987729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:42:58.969585105Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:42:59.717940624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="454 kB" avg_comp="152 kB" total_uncomp="4.9 MB" avg_uncomp="1.6 MB" idle=1 full=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:42:59.723280389Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:43:01.240832151Z caller=checkpoint.go:378 msg="writing series" size="763 kB"
level=warn ts=2026-02-16T18:43:03.071286284Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:43:03.781979429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:43:03.786730086Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T18:43:05.043229079Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T18:43:05.734395565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:43:05.738424128Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:43:06.19371842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:43:06.198087578Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:43:07.161753433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:43:07.165986645Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T18:43:08.071644136Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:43:08.846707939Z caller=checkpoint.go:378 msg="writing series" size="101 kB"
ts=2026-02-16T18:43:11.280004203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=debug ts=2026-02-16T18:43:12.649543299Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=info ts=2026-02-16T18:43:13.047904526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="910 B" avg_comp="455 B" total_uncomp="694 B" avg_uncomp="347 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:43:13.052545791Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:43:13.053281867Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:43:13.054363871Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:43:13.071863426Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:43:13.071886988Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:43:13.071894376Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:43:13.071913798Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:43:13.149198452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:43:13.156124863Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:43:14.087898754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:43:14.091576663Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:43:16.451990989Z caller=checkpoint.go:378 msg="writing series" size="111 kB"
ts=2026-02-16T18:43:17.408399482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:52398"
level=warn ts=2026-02-16T18:43:18.071207769Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:43:19.491106391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:48224"
level=info ts=2026-02-16T18:43:20.06093115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:43:20.065020419Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:43:20.254726494Z caller=checkpoint.go:378 msg="writing series" size="469 kB"
level=debug ts=2026-02-16T18:43:20.254750057Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=8
level=info ts=2026-02-16T18:43:20.398893482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:43:20.403285891Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:43:22.047495874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.4 kB" avg_comp="1.1 kB" total_uncomp="9.5 kB" avg_uncomp="3.2 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:43:22.051099544Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T18:43:23.071239341Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:43:23.302273751Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:43:24.057383733Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T18:43:24.803198042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4c0d02f8eff28ba8 immediate=false num_chunks=1 total_comp="139 kB" avg_comp="139 kB" total_uncomp="673 kB" avg_uncomp="673 kB" full=1 labels="{stream=\"gitlab-rep-qtrv\"}"
level=error ts=2026-02-16T18:43:24.807629073Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4c0d02f8eff28ba8/19c67b2c8ae%3A19c67bb3712%3A238c6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qtrv\"}"
level=info ts=2026-02-16T18:43:25.240799408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:43:25.244581392Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T18:43:26.641285076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:43:26.646201173Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:43:27.36436628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="294 kB" avg_uncomp="294 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:43:27.368268545Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T18:43:27.860865892Z caller=checkpoint.go:378 msg="writing series" size="79 kB"
level=warn ts=2026-02-16T18:43:28.071711031Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:43:29.352735642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:43:29.356971049Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:43:31.065485514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:43:31.068920365Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:43:31.663814683Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=info ts=2026-02-16T18:43:32.248411905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="715 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:43:32.251807464Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T18:43:33.071720042Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:43:35.030384224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:43:35.034595477Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T18:43:35.466020639Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T18:43:35.484177653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:43:35.488088972Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T18:43:38.071624128Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:43:39.268894837Z caller=checkpoint.go:378 msg="writing series" size="183 kB"
level=info ts=2026-02-16T18:43:40.664008418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:43:40.669176572Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:43:40.669197584Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:43:40.669219975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:43:40.672063004Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
ts=2026-02-16T18:43:41.282769944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T18:43:42.680063091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:43:42.68382523Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:43:43.071087162Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:43:43.071119466Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:43:43.071128043Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:43:43.071123197Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:43:43.072178117Z caller=checkpoint.go:390 msg="flushing series" totalSize="296 kB" series=5
level=info ts=2026-02-16T18:43:43.075499Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-02-16T18:43:43.076578507Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.005290393s
level=info ts=2026-02-16T18:43:44.990681261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:43:44.99462421Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:43:45.192392589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:43:45.195636021Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:43:45.480509549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:43:45.484439598Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:43:46.051721068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:43:46.055015339Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:43:46.744677208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:43:46.748092298Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:43:47.099193281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:43:47.102982987Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:43:48.071288438Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:43:48.288387649Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:43:50.210579266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:43:50.214049473Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:43:53.018209188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="454 kB" avg_comp="152 kB" total_uncomp="4.9 MB" avg_uncomp="1.6 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:43:53.022832306Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:43:53.071216344Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:43:53.09737195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:43:53.100767238Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T18:43:53.75138721Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:43:55.970463256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:43:55.974871007Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T18:43:58.071065365Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:43:59.482822794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:43:59.488509268Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:44:02.234420277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:44:02.239535307Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:44:03.07109277Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:44:04.067749604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="910 B" avg_comp="455 B" total_uncomp="694 B" avg_uncomp="347 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:44:04.071239272Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
ts=2026-02-16T18:44:06.492509585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33912"
level=info ts=2026-02-16T18:44:07.075375208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:44:07.081775068Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:44:08.042907393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:44:08.046590307Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:44:08.071962871Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:44:09.633495943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:44:09.637096151Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T18:44:10.517023387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:44:10.520458731Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:44:10.520482771Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:44:10.520506108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:44:10.52380955Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T18:44:11.285034168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:44:11.571216146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.4 kB" avg_comp="1.1 kB" total_uncomp="9.5 kB" avg_uncomp="3.2 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:44:11.574587036Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T18:44:13.054262719Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:44:13.054298403Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:44:13.07175902Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:44:13.071785814Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T18:44:13.071774447Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:44:13.071783553Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T18:44:13.07179398Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:44:13.071919487Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=debug ts=2026-02-16T18:44:13.076198922Z caller=checkpoint.go:378 msg="writing series" size="7.1 MB"
level=debug ts=2026-02-16T18:44:13.076210367Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.1 MB" series=1
level=info ts=2026-02-16T18:44:14.530210079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4c0d02f8eff28ba8 immediate=false num_chunks=1 total_comp="139 kB" avg_comp="139 kB" total_uncomp="673 kB" avg_uncomp="673 kB" full=1 labels="{stream=\"gitlab-rep-qtrv\"}"
level=error ts=2026-02-16T18:44:14.534060013Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4c0d02f8eff28ba8/19c67b2c8ae%3A19c67bb3712%3A238c6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qtrv\"}"
level=debug ts=2026-02-16T18:44:16.875839462Z caller=checkpoint.go:378 msg="writing series" size="427 kB"
ts=2026-02-16T18:44:17.413331457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:55690"
level=warn ts=2026-02-16T18:44:18.072044484Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:44:19.268107985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:44:19.274188113Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-02-16T18:44:19.495751169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:48054"
level=info ts=2026-02-16T18:44:19.666577983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:44:19.670233509Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:44:20.38402792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:44:20.387446582Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:44:20.552949582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:44:20.556652335Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:44:20.678261912Z caller=checkpoint.go:378 msg="writing series" size="143 kB"
level=debug ts=2026-02-16T18:44:21.584158087Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:44:23.07188835Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:44:23.085039378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:44:23.088900402Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:44:23.138970926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:44:23.142800852Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:44:24.159397271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="294 kB" avg_uncomp="294 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:44:24.163124535Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T18:44:24.48108267Z caller=checkpoint.go:378 msg="writing series" size="121 kB"
level=debug ts=2026-02-16T18:44:25.547735431Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:44:25.549561732Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T18:44:25.678033542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:44:25.681690245Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:44:28.071098043Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:44:28.284575028Z caller=checkpoint.go:378 msg="writing series" size="163 kB"
level=info ts=2026-02-16T18:44:29.993049156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:44:29.998140459Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:44:30.318801087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:44:30.323127267Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T18:44:32.087047018Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=info ts=2026-02-16T18:44:32.874351919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:44:32.878586512Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:44:33.071357648Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:44:33.76619542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:44:33.769745378Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:44:35.889208213Z caller=checkpoint.go:378 msg="writing series" size="510 B"
level=warn ts=2026-02-16T18:44:38.070971127Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:44:38.074118608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:44:38.077992438Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:44:38.078017083Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:44:38.078038454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:44:38.081077055Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:44:39.668985105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:44:39.672398435Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:44:39.692788112Z caller=checkpoint.go:378 msg="writing series" size="208 kB"
level=debug ts=2026-02-16T18:44:39.692813086Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
ts=2026-02-16T18:44:41.28800422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:44:41.806933964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:44:41.811042944Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:44:42.468476125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:44:42.471895119Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:44:42.624195275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:44:42.627515123Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:44:43.071258738Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:44:43.071279647Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:44:43.071301774Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:44:43.071310719Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:44:43.495170573Z caller=checkpoint.go:378 msg="writing series" size="917 B"
level=info ts=2026-02-16T18:44:43.535270582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:44:43.540108933Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T18:44:43.608388691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:44:43.611887718Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:44:43.781362525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="454 kB" avg_comp="152 kB" total_uncomp="4.9 MB" avg_uncomp="1.6 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:44:43.786116845Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:44:46.343763478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:44:46.347130417Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:44:46.667095965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:44:46.670528592Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:44:47.298381503Z caller=checkpoint.go:378 msg="writing series" size="430 kB"
ts=2026-02-16T18:44:47.416406867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:60672"
level=warn ts=2026-02-16T18:44:48.071887964Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:44:51.10128947Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=info ts=2026-02-16T18:44:51.456220151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:44:51.463105163Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:44:51.463130742Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:44:51.463174932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:44:51.468795998Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:44:51.843676801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:44:51.847517572Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=warn ts=2026-02-16T18:44:53.071619376Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:44:53.141776372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:44:53.144955241Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:44:53.770929917Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:44:54.903826511Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
level=info ts=2026-02-16T18:44:55.320082957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.4 kB" avg_comp="1.1 kB" total_uncomp="9.5 kB" avg_uncomp="3.2 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:44:55.323446246Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T18:44:58.071939956Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:44:58.706773137Z caller=checkpoint.go:378 msg="writing series" size="274 kB"
level=info ts=2026-02-16T18:44:59.46513701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:44:59.470123672Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:44:59.488198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4c0d02f8eff28ba8 immediate=false num_chunks=1 total_comp="139 kB" avg_comp="139 kB" total_uncomp="673 kB" avg_uncomp="673 kB" full=1 labels="{stream=\"gitlab-rep-qtrv\"}"
level=error ts=2026-02-16T18:44:59.49159445Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4c0d02f8eff28ba8/19c67b2c8ae%3A19c67bb3712%3A238c6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qtrv\"}"
level=info ts=2026-02-16T18:45:01.62723347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="910 B" avg_comp="455 B" total_uncomp="694 B" avg_uncomp="347 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:45:01.632831147Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:45:01.699213015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:45:01.706086049Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:45:02.509107068Z caller=checkpoint.go:378 msg="writing series" size="809 kB"
level=debug ts=2026-02-16T18:45:02.50913525Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.8 MB" series=6
level=warn ts=2026-02-16T18:45:03.071083479Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:45:03.32076588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:45:03.325909279Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:45:06.312748118Z caller=checkpoint.go:378 msg="writing series" size="411 kB"
level=info ts=2026-02-16T18:45:07.270895135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:45:07.274796729Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:45:07.274820213Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:45:07.274847427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:45:07.276517018Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:45:08.071443386Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:45:09.079161414Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:45:10.114998541Z caller=checkpoint.go:378 msg="writing series" size="589 kB"
ts=2026-02-16T18:45:11.294519582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:45:11.606179946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:45:11.609655939Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T18:45:13.054235742Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:45:13.055296922Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T18:45:13.055572216Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-02-16T18:45:13.055635651Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1968074 n=1
level=debug ts=2026-02-16T18:45:13.055694538Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1968074 n=1
level=warn ts=2026-02-16T18:45:13.071058927Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:45:13.071057053Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:45:13.071088479Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:45:13.071097828Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:45:13.917822589Z caller=checkpoint.go:378 msg="writing series" size="965 kB"
level=debug ts=2026-02-16T18:45:13.917852069Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=3
level=info ts=2026-02-16T18:45:14.051581194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:45:14.055754449Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:45:14.407454201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:45:14.411084501Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:45:15.39341287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:45:15.396720082Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:45:17.720990587Z caller=checkpoint.go:378 msg="writing series" size="9.2 kB"
level=warn ts=2026-02-16T18:45:18.071893435Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:45:18.398088551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:45:18.401526379Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:45:19.305216789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:45:19.308969615Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-02-16T18:45:19.500187441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:37486"
level=info ts=2026-02-16T18:45:19.575715385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="294 kB" avg_uncomp="294 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:45:19.579249473Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:45:19.826731734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:45:19.830242425Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:45:21.523356761Z caller=checkpoint.go:378 msg="writing series" size="339 kB"
level=warn ts=2026-02-16T18:45:23.07161884Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:45:23.283385864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:45:23.287674323Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:45:25.326668007Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T18:45:25.891515318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:45:25.895302861Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:45:26.060366295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:45:26.064321326Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:45:26.608863162Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:45:27.594192004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:45:27.5977793Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:45:28.071543152Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:45:29.128802747Z caller=checkpoint.go:378 msg="writing series" size="9.1 kB"
level=info ts=2026-02-16T18:45:29.824031005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:45:29.827539611Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:45:30.561201754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:45:30.564661413Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:45:30.610742056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:45:30.614232648Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:45:31.583891154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:45:31.589588201Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:45:32.934808967Z caller=checkpoint.go:378 msg="writing series" size="3.4 MB"
level=debug ts=2026-02-16T18:45:32.934837511Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.8 MB" series=5
level=warn ts=2026-02-16T18:45:33.07120105Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:45:33.818506881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:45:33.828758678Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:45:33.833815157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="454 kB" avg_comp="152 kB" total_uncomp="4.9 MB" avg_uncomp="1.6 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:45:33.837539725Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:45:36.734700084Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB"
level=info ts=2026-02-16T18:45:36.997562317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:45:37.001281604Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:45:38.071317039Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:45:38.088478714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.4 kB" avg_comp="1.1 kB" total_uncomp="9.5 kB" avg_uncomp="3.2 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:45:38.092009312Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T18:45:40.537741539Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
ts=2026-02-16T18:45:41.297777927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:45:42.316888349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.7 kB" avg_comp="3.7 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:45:42.320902787Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:45:42.841360461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:45:42.846395353Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T18:45:43.071279264Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:45:43.07127303Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:45:43.07130473Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:45:43.071313519Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:45:43.369724814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:45:43.373830722Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:45:43.373854939Z caller=flush.go:240 component=ingester loop=10 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:45:43.373884933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:45:43.377347359Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=debug ts=2026-02-16T18:45:44.339984857Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
ts=2026-02-16T18:45:47.420821787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:34942"
level=info ts=2026-02-16T18:45:47.790612179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:45:47.794222534Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:45:48.071658392Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:45:48.143238434Z caller=checkpoint.go:378 msg="writing series" size="886 kB"
level=debug ts=2026-02-16T18:45:48.998917051Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:45:50.97529691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:45:50.980545429Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:45:51.945539353Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T18:45:52.075144397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="345 B" total_uncomp="1.0 kB" avg_uncomp="344 B" idle=1 full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:45:52.079035169Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:45:52.079061339Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:45:52.079086739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="345 B" total_uncomp="1.0 kB" avg_uncomp="344 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:45:52.082756941Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:45:53.070965097Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:45:54.814761629Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:45:54.908072026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4c0d02f8eff28ba8 immediate=false num_chunks=1 total_comp="139 kB" avg_comp="139 kB" total_uncomp="673 kB" avg_uncomp="673 kB" full=1 labels="{stream=\"gitlab-rep-qtrv\"}"
level=error ts=2026-02-16T18:45:54.912181528Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4c0d02f8eff28ba8/19c67b2c8ae%3A19c67bb3712%3A238c6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qtrv\"}"
level=error ts=2026-02-16T18:45:54.912202772Z caller=flush.go:240 component=ingester loop=8 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:45:54.912227066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="384 kB" avg_uncomp="384 kB" idle=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:45:54.915263166Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=debug ts=2026-02-16T18:45:55.748757943Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=debug ts=2026-02-16T18:45:55.748783916Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-02-16T18:45:58.071284548Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:45:59.55153806Z caller=checkpoint.go:378 msg="writing series" size="165 kB"
level=info ts=2026-02-16T18:46:01.203372852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:46:01.208267282Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T18:46:01.549392855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:46:01.554353462Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:46:03.071109768Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:46:03.354092896Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=info ts=2026-02-16T18:46:03.938216162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:46:03.947383213Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T18:46:07.157408999Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=info ts=2026-02-16T18:46:07.429690296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:46:07.435531626Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T18:46:08.071595431Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:46:08.08373982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="345 B" total_uncomp="1.0 kB" avg_uncomp="344 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:46:08.087974731Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:46:08.397046893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:46:08.401197555Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:46:08.468273726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:46:08.472077581Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:46:10.119912403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:46:10.124135862Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:46:10.959621449Z caller=checkpoint.go:378 msg="writing series" size="228 kB"
ts=2026-02-16T18:46:11.300544302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:46:12.563686428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:46:12.568020224Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=debug ts=2026-02-16T18:46:13.053694873Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:46:13.054746324Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:46:13.071015747Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:46:13.071040931Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:46:13.071049283Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:46:13.071063254Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:46:14.355361581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:46:14.358923208Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:46:14.763061163Z caller=checkpoint.go:378 msg="writing series" size="5.0 kB"
level=info ts=2026-02-16T18:46:14.998722105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="454 kB" avg_comp="152 kB" total_uncomp="4.9 MB" avg_uncomp="1.6 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:46:15.002574393Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:46:15.600018584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:46:15.604018361Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:46:15.752624262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="294 kB" avg_uncomp="294 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:46:15.756455196Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:46:16.030067943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:46:16.033376292Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:46:17.796276917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:46:17.800196474Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:46:18.071330344Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:46:18.566136062Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
ts=2026-02-16T18:46:19.504963639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:34242"
level=info ts=2026-02-16T18:46:20.890926158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:46:20.894674043Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:46:21.1213742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:46:21.125552012Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T18:46:22.368113891Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=info ts=2026-02-16T18:46:22.904737954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="917 B" avg_comp="917 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:46:22.908711146Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:46:23.070535475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:46:23.071587337Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-02-16T18:46:23.073895847Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:46:24.016669858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:46:24.023277991Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:46:25.652436292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:46:25.661454303Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:46:25.906209429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:46:25.9109295Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T18:46:26.162675267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:46:26.166436955Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:46:26.171785752Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-02-16T18:46:27.928733649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:46:27.932848338Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:46:28.071262215Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:46:28.464925659Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:46:29.973746406Z caller=checkpoint.go:378 msg="writing series" size="293 kB"
level=info ts=2026-02-16T18:46:33.046300126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:46:33.049962699Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:46:33.071126967Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:46:33.452202569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.9 kB" avg_comp="1.3 kB" total_uncomp="9.5 kB" avg_uncomp="3.2 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:46:33.455973621Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T18:46:33.776624136Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=debug ts=2026-02-16T18:46:33.898559867Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:46:37.417579757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="339 B" total_uncomp="1.0 kB" avg_uncomp="344 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:46:37.422340007Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:46:37.580083637Z caller=checkpoint.go:378 msg="writing series" size="98 kB"
level=warn ts=2026-02-16T18:46:38.071647399Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:46:41.30344613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:46:41.30998898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:46:41.315315234Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T18:46:41.382550965Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T18:46:42.239076579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:46:42.245663793Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:46:43.071534354Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:46:43.071570857Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:46:43.071581681Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:46:43.07159467Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:46:44.975679938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:46:44.979801618Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=debug ts=2026-02-16T18:46:45.185439927Z caller=checkpoint.go:378 msg="writing series" size="9.7 kB"
ts=2026-02-16T18:46:47.425823469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:54022"
level=warn ts=2026-02-16T18:46:48.071267593Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:46:48.987847863Z caller=checkpoint.go:378 msg="writing series" size="170 kB"
level=debug ts=2026-02-16T18:46:48.987877782Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=14
level=debug ts=2026-02-16T18:46:52.791362829Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=warn ts=2026-02-16T18:46:53.071920686Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:46:54.24028488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:46:54.24432784Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:46:54.597415443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:46:54.601168526Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:46:55.440096731Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:46:55.46435239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:46:55.468293212Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:46:55.982714147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:46:55.990247157Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:46:56.594039855Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=warn ts=2026-02-16T18:46:58.071197031Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:47:00.396444374Z caller=checkpoint.go:378 msg="writing series" size="900 B"
level=info ts=2026-02-16T18:47:00.456683696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:47:00.460311895Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:47:03.050734375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:47:03.055083567Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:47:03.071125727Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:47:03.379457827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:47:03.382782584Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:47:03.719809797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:47:03.726143602Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T18:47:04.199905237Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T18:47:04.689916657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:47:04.69359271Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:47:05.66358845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:47:05.66730088Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:47:06.799374216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:47:06.804299675Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:47:07.231587281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:47:07.236308598Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=debug ts=2026-02-16T18:47:08.002560955Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T18:47:08.071091259Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:47:08.345134539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="454 kB" avg_comp="152 kB" total_uncomp="4.9 MB" avg_uncomp="1.6 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:47:08.349468244Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:47:08.487430343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:47:08.49163573Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:47:09.401604418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="294 kB" avg_uncomp="294 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:47:09.405485745Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
ts=2026-02-16T18:47:11.306072613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=debug ts=2026-02-16T18:47:11.804760573Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=debug ts=2026-02-16T18:47:13.054171459Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:47:13.055244749Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:47:13.071749504Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:47:13.071785041Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T18:47:13.071784998Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:47:13.071795799Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:47:13.674792997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:47:13.678219465Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:47:13.67824077Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:47:13.678262354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:47:13.68093561Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:47:14.878090737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:47:14.882041718Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:47:15.447559481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:47:15.450896916Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T18:47:15.608355632Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=info ts=2026-02-16T18:47:17.568916474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:47:17.572319231Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:47:17.84712235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:47:17.850608729Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:47:17.850627458Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:47:17.85064938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:47:17.853332804Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:47:18.071677896Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:47:19.410425014Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=debug ts=2026-02-16T18:47:19.976889397Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:47:21.26463594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="345 B" total_uncomp="1.0 kB" avg_uncomp="344 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:47:21.26810638Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:47:22.955998137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:47:22.962031782Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:47:23.071805207Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:47:23.213191676Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=debug ts=2026-02-16T18:47:27.016242908Z caller=checkpoint.go:378 msg="writing series" size="231 kB"
level=info ts=2026-02-16T18:47:27.593994834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:47:27.598649703Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:47:28.071172023Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:47:29.830813293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:47:29.836318346Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:47:30.819523382Z caller=checkpoint.go:378 msg="writing series" size="997 kB"
level=debug ts=2026-02-16T18:47:30.819549208Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=11
level=info ts=2026-02-16T18:47:31.885459923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:47:31.890630257Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:47:32.470962315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.9 kB" avg_comp="1.3 kB" total_uncomp="9.5 kB" avg_uncomp="3.2 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:47:32.474422042Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T18:47:33.071244721Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:47:34.621620676Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=debug ts=2026-02-16T18:47:34.772237112Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:47:35.972291837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:47:35.975592776Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T18:47:36.503883114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:52234"
level=info ts=2026-02-16T18:47:37.39008696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:47:37.393692721Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:47:37.509366205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:47:37.512991295Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=warn ts=2026-02-16T18:47:38.071651435Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:47:38.42473275Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T18:47:41.088280581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:47:41.092819163Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-02-16T18:47:41.30930418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=debug ts=2026-02-16T18:47:42.228416853Z caller=checkpoint.go:378 msg="writing series" size="2.4 MB"
level=debug ts=2026-02-16T18:47:42.228444294Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.4 MB" series=3
level=info ts=2026-02-16T18:47:42.933174571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:47:42.936969195Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:47:42.936990073Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:47:42.937011324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:47:42.941332815Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T18:47:43.071049092Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:47:43.071053006Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:47:43.071076151Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:47:43.071084Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:47:44.206775969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:47:44.210032862Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:47:44.21005594Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:47:44.210078559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:47:44.212870207Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:47:45.086278609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:47:45.089788913Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T18:47:46.03024056Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
level=info ts=2026-02-16T18:47:47.56663137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:47:47.570601003Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T18:47:48.071169455Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:47:49.510291358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:37656"
level=debug ts=2026-02-16T18:47:49.848164277Z caller=checkpoint.go:378 msg="writing series" size="25 MB"
level=debug ts=2026-02-16T18:47:49.848191169Z caller=checkpoint.go:390 msg="flushing series" totalSize="25 MB" series=2
level=debug ts=2026-02-16T18:47:50.597690009Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:47:50.622720737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:47:50.627353307Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:47:53.071702051Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:47:53.236037304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="587 B" avg_comp="587 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:47:53.240286987Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:47:53.636343963Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=info ts=2026-02-16T18:47:54.668731233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:47:54.672067207Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T18:47:54.862720126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:47:54.866615075Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:47:56.160438054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:47:56.163834473Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T18:47:56.163853919Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:47:56.16387569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:47:56.166728901Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:47:56.399375499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:47:56.402773379Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:47:57.439053795Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-02-16T18:47:58.071381142Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:47:58.506623597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:47:58.51103732Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:47:59.498791094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:47:59.502428963Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:48:00.591643143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:48:00.59545178Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:48:00.845082701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="98 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:48:00.848579372Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:48:01.241836347Z caller=checkpoint.go:378 msg="writing series" size="533 kB"
level=info ts=2026-02-16T18:48:02.68703507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:48:02.691260909Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:48:02.691279973Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:48:02.691300627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:48:02.69425575Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:48:03.071599119Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:48:03.131755953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="345 B" total_uncomp="1.0 kB" avg_uncomp="344 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:48:03.136051905Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:48:04.700831985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:48:04.705118876Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=debug ts=2026-02-16T18:48:05.044623797Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T18:48:05.199203191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:48:05.205546952Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:48:05.400998299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:48:05.404493311Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:48:05.789500794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="294 kB" avg_uncomp="294 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:48:05.793193208Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:48:06.45017377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:48:06.453690663Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-02-16T18:48:06.506508925Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60468"
level=info ts=2026-02-16T18:48:07.754202222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:48:07.757829371Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:48:08.027490462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="477 kB" avg_comp="159 kB" total_uncomp="4.9 MB" avg_uncomp="1.6 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:48:08.03145353Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:48:08.031475341Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:48:08.031502418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="477 kB" avg_comp="159 kB" total_uncomp="4.9 MB" avg_uncomp="1.6 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:48:08.035955249Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:48:08.071203295Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:48:08.847411702Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
level=info ts=2026-02-16T18:48:09.422204237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:48:09.42925332Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T18:48:11.311749989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=debug ts=2026-02-16T18:48:12.649719496Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=debug ts=2026-02-16T18:48:13.054197892Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:48:13.055255596Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:48:13.071531449Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:48:13.071550618Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:48:13.071558865Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:48:13.071574968Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:48:16.453247498Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
level=info ts=2026-02-16T18:48:16.939640211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:48:16.944934796Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:48:16.944959289Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:48:16.944986232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:48:16.949217288Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:48:18.071528618Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:48:19.240383764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:48:19.243779134Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:48:19.589719157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.9 kB" avg_comp="1.3 kB" total_uncomp="9.5 kB" avg_uncomp="3.2 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:48:19.593073998Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T18:48:19.593099168Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:48:19.593126852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.9 kB" avg_comp="1.3 kB" total_uncomp="9.5 kB" avg_uncomp="3.2 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:48:19.596009347Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T18:48:20.256118325Z caller=checkpoint.go:378 msg="writing series" size="469 kB"
level=debug ts=2026-02-16T18:48:20.256144035Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-02-16T18:48:20.611912293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:48:20.615688915Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T18:48:20.806310645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="477 kB" avg_comp="159 kB" total_uncomp="4.9 MB" avg_uncomp="1.6 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:48:20.809923098Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:48:21.866682091Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:48:22.418030345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:48:22.422587837Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=warn ts=2026-02-16T18:48:23.071798242Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:48:24.058903648Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T18:48:24.946364669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:48:24.949966323Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:48:27.80121782Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:48:27.803381683Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T18:48:27.861913614Z caller=checkpoint.go:378 msg="writing series" size="89 kB"
level=warn ts=2026-02-16T18:48:28.071324527Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:48:31.664210974Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=info ts=2026-02-16T18:48:31.74531114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:48:31.751574035Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:48:31.751597731Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:48:31.751622319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:48:31.755642846Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T18:48:33.071187242Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:48:33.74799774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:48:33.753071182Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:48:33.879495668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:48:33.883137066Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:48:33.8831779Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:48:33.883200368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:48:33.885990262Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:48:34.063770005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="715 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:48:34.067090263Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:48:35.466738737Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T18:48:35.532851632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" idle=1 full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:48:35.538812992Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:48:36.016572509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:48:36.020130936Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T18:48:38.071455939Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:48:39.060685218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:48:39.06431513Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:48:39.064336073Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:48:39.064357161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:48:39.067260876Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:48:39.269581558Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=info ts=2026-02-16T18:48:40.721089639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:48:40.725123376Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:48:41.092279999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:48:41.095711329Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:48:41.095732839Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:48:41.095754079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:48:41.098499018Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
ts=2026-02-16T18:48:41.314189256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:48:42.643027751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:48:42.646524055Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:48:43.071178657Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:48:43.071216271Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:48:43.071225444Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:48:43.071235071Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:48:43.072285882Z caller=checkpoint.go:390 msg="flushing series" totalSize="307 kB" series=5
level=info ts=2026-02-16T18:48:43.077301081Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-02-16T18:48:43.078276408Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.006240198s
level=info ts=2026-02-16T18:48:44.605027678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="48 kB" avg_uncomp="24 kB" idle=1 full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:48:44.608682138Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:48:45.337932241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:48:45.342522095Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T18:48:45.837779444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="477 kB" avg_comp="159 kB" total_uncomp="4.9 MB" avg_uncomp="1.6 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:48:45.843220917Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:48:46.230422245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:48:46.233967713Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:48:46.233989577Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:48:46.234011787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:48:46.236792809Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:48:46.846715735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="345 B" total_uncomp="1.0 kB" avg_uncomp="344 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:48:46.850062334Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:48:47.919513725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:48:47.923504536Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:48:47.923525598Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:48:47.92354672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:48:47.926963825Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:48:48.071590295Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:48:48.147911667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:48:48.151218972Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:48:49.97626455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:48:49.979770629Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:48:50.661985205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:48:50.665311266Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:48:50.665337651Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:48:50.665366743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:48:50.668078412Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:48:51.508306799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:48:51.511767376Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T18:48:51.981084184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:48:51.98433171Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:48:53.071848249Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:48:53.254297344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" idle=1 full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:48:53.257697337Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:48:54.159967565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:48:54.163862067Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:48:54.987661283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="45 kB" avg_comp="22 kB" total_uncomp="305 kB" avg_uncomp="152 kB" idle=1 full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:48:54.991339354Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:48:54.9913578Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:48:54.991378207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="45 kB" avg_comp="22 kB" total_uncomp="305 kB" avg_uncomp="152 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:48:54.994207231Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:48:55.011273065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:48:55.014440298Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:48:55.327429195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:48:55.331983414Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:48:55.466513606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:48:55.472808737Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:48:56.481802253Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:48:56.898007488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:48:56.901512668Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:48:57.847727485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:48:57.85266704Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:48:58.071555981Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:48:59.238748503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:48:59.242125606Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T18:49:03.07113571Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:49:03.661746467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:49:03.666301886Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:49:05.979724008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:49:05.983561844Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=info ts=2026-02-16T18:49:07.660194986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="45 kB" avg_comp="22 kB" total_uncomp="305 kB" avg_uncomp="152 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:49:07.66406962Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:49:07.889715475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:49:07.893125451Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:49:08.071690329Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:49:09.194561179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:49:09.198280925Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:49:09.19830205Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:49:09.198322569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:49:09.201091153Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T18:49:11.316952658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:49:12.022191057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:49:12.025599913Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T18:49:13.054185367Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:49:13.055195355Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:49:13.071591949Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T18:49:13.071633918Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:49:13.071657544Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:49:13.071666194Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:49:13.071684903Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:49:13.071792139Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=debug ts=2026-02-16T18:49:13.076188849Z caller=checkpoint.go:378 msg="writing series" size="7.1 MB"
level=debug ts=2026-02-16T18:49:13.076211158Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.1 MB" series=1
level=info ts=2026-02-16T18:49:13.77718522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:49:13.780511405Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T18:49:15.543138034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:49:15.54744362Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T18:49:15.847256563Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:49:16.822951587Z caller=checkpoint.go:378 msg="writing series" size="312 kB"
level=info ts=2026-02-16T18:49:16.868199334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:49:16.871566049Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T18:49:18.071519565Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:49:19.211207559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:49:19.214571476Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:49:19.477383711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:49:19.480827577Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
ts=2026-02-16T18:49:19.515320804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:57704"
level=debug ts=2026-02-16T18:49:20.572309668Z caller=checkpoint.go:378 msg="writing series" size="143 kB"
level=debug ts=2026-02-16T18:49:20.683881002Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:49:21.348253822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:49:21.351646632Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:49:21.631429356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:49:21.634838005Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T18:49:23.07147004Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:49:23.270331154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:49:23.27372261Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:49:24.322716308Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
level=info ts=2026-02-16T18:49:24.9053474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:49:24.908859458Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:49:25.764515528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:49:25.767955427Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:49:27.968735091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:49:27.972089279Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:49:28.071565878Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:49:28.072638306Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=info ts=2026-02-16T18:49:28.743495901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="477 kB" avg_comp="159 kB" total_uncomp="4.9 MB" avg_uncomp="1.6 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:49:28.747200757Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:49:29.432413411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="45 kB" avg_comp="22 kB" total_uncomp="305 kB" avg_uncomp="152 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:49:29.435943424Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:49:30.056888305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="48 kB" avg_uncomp="24 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:49:30.060477119Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:49:31.599533393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:49:31.603583796Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=debug ts=2026-02-16T18:49:31.822224519Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=warn ts=2026-02-16T18:49:33.070979169Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:49:34.785894113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:49:34.791387882Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:49:35.572554835Z caller=checkpoint.go:378 msg="writing series" size="897 B"
level=warn ts=2026-02-16T18:49:38.071254922Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:49:38.136369918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:49:38.141491539Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:49:39.32283276Z caller=checkpoint.go:378 msg="writing series" size="208 kB"
level=info ts=2026-02-16T18:49:39.857199407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:49:39.86057299Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:49:40.035873195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:49:40.039246011Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:49:40.039267524Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:49:40.039289996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:49:40.042909121Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T18:49:41.319515392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:49:41.500411315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:49:41.503850746Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:49:42.755456899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="345 B" total_uncomp="1.0 kB" avg_uncomp="344 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:49:42.758912448Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:49:43.070957222Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:49:43.070958242Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:49:43.070986431Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:49:43.070996894Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:49:43.072107731Z caller=checkpoint.go:378 msg="writing series" size="917 B"
level=debug ts=2026-02-16T18:49:45.795022788Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:49:46.82288025Z caller=checkpoint.go:378 msg="writing series" size="338 kB"
level=debug ts=2026-02-16T18:49:46.822905275Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=9
level=info ts=2026-02-16T18:49:47.813551592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:49:47.817615449Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:49:48.07163839Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:49:50.473808853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:49:50.47908667Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:49:50.572461369Z caller=checkpoint.go:378 msg="writing series" size="174 kB"
level=info ts=2026-02-16T18:49:52.036920374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:49:52.043336689Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:49:53.071788356Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:49:53.250336416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:49:53.254072224Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:49:53.80119661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:49:53.804554977Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=info ts=2026-02-16T18:49:54.212229958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:49:54.215798124Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:49:54.323121015Z caller=checkpoint.go:378 msg="writing series" size="282 kB"
level=info ts=2026-02-16T18:49:56.152753313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:49:56.156146633Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:49:56.729814884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:49:56.733350303Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:49:56.822600876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:49:56.826339534Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-02-16T18:49:58.07186213Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:49:58.0720734Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=debug ts=2026-02-16T18:50:01.822698018Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB"
level=debug ts=2026-02-16T18:50:01.822724028Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=4
level=warn ts=2026-02-16T18:50:03.071645559Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:50:03.671169571Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:50:04.754974682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:50:04.758440034Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:50:04.758472614Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:50:04.758495726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:50:04.761213899Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:50:05.172373609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:50:05.176660986Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:50:05.573096047Z caller=checkpoint.go:378 msg="writing series" size="431 kB"
ts=2026-02-16T18:50:06.512330249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53530"
level=info ts=2026-02-16T18:50:07.680677447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:50:07.685222456Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T18:50:08.071450306Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:50:09.323284988Z caller=checkpoint.go:378 msg="writing series" size="819 kB"
level=debug ts=2026-02-16T18:50:09.323310968Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=2
ts=2026-02-16T18:50:11.32244435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=debug ts=2026-02-16T18:50:12.052087906Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:50:13.05334281Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:50:13.054385872Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:50:13.071880815Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:50:13.071909981Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:50:13.071920059Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:50:13.071940513Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:50:13.073398675Z caller=checkpoint.go:378 msg="writing series" size="936 kB"
level=info ts=2026-02-16T18:50:14.994465129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:50:14.998474926Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:50:15.944189251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:50:15.948293097Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:50:16.323473663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:50:16.327251524Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:50:16.822687255Z caller=checkpoint.go:378 msg="writing series" size="9.5 kB"
level=info ts=2026-02-16T18:50:17.840494548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:50:17.843766497Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:50:18.027219507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:50:18.03105591Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=warn ts=2026-02-16T18:50:18.071275029Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:50:19.302702125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:50:19.306233222Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:50:20.037494973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:50:20.040892294Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:50:20.572806864Z caller=checkpoint.go:378 msg="writing series" size="457 kB"
level=debug ts=2026-02-16T18:50:20.572839877Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=3
level=warn ts=2026-02-16T18:50:23.071358379Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:50:23.740201173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:50:23.743687744Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:50:24.022482076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:50:24.025944048Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:50:24.032981889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="48 kB" avg_uncomp="24 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:50:24.035926378Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:50:24.322678752Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T18:50:24.55105135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:50:24.554335974Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:50:25.707531916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="45 kB" avg_comp="22 kB" total_uncomp="305 kB" avg_uncomp="152 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:50:25.71099937Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:50:26.579577548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:50:26.583639901Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:50:27.65825475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:50:27.661586059Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T18:50:28.071673328Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:50:28.072792903Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-02-16T18:50:28.113904758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="477 kB" avg_comp="159 kB" total_uncomp="4.9 MB" avg_uncomp="1.6 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:50:28.117528164Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:50:31.829464071Z caller=checkpoint.go:378 msg="writing series" size="5.9 MB"
level=debug ts=2026-02-16T18:50:31.829490694Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.9 MB" series=3
level=info ts=2026-02-16T18:50:32.009353202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:50:32.013117471Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:50:32.013139982Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:50:32.013188254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:50:32.016732016Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:50:33.071904246Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:50:33.137065756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="345 B" total_uncomp="1.0 kB" avg_uncomp="344 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:50:33.140694047Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:50:35.345401324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:50:35.352191304Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:50:35.469584022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:50:35.473123077Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:50:35.572371116Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB"
level=info ts=2026-02-16T18:50:37.543724907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:50:37.54740005Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:50:38.071198704Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:50:39.322890457Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
level=info ts=2026-02-16T18:50:40.408115115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:50:40.412499364Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=info ts=2026-02-16T18:50:40.450668677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:50:40.454629264Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
ts=2026-02-16T18:50:41.325139491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T18:50:43.071107639Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:50:43.071126128Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:50:43.071160308Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:50:43.071184117Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:50:43.072227881Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T18:50:43.40228639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:50:43.408839137Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:50:44.569265853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.7 kB" avg_comp="3.7 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:50:44.572773397Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:50:44.650039402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:50:44.653449343Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:50:46.28498723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:50:46.288463092Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:50:46.823048394Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T18:50:46.823073331Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-02-16T18:50:46.908608308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:50:46.91182213Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T18:50:47.330771036Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:50:47.630556803Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:50:47.689763021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:50:47.693819923Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T18:50:48.071182628Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:50:50.572767236Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=warn ts=2026-02-16T18:50:53.070973857Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:50:54.32248839Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T18:50:55.824622032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:50:55.828300766Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:50:57.185202723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="917 B" avg_comp="917 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:50:57.188571955Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:50:58.071709301Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:50:58.072923626Z caller=checkpoint.go:378 msg="writing series" size="208 kB"
level=debug ts=2026-02-16T18:51:01.822330048Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=warn ts=2026-02-16T18:51:03.071186337Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:51:04.726505692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:51:04.730633202Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:51:05.29980225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:51:05.30323582Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T18:51:05.573057121Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=info ts=2026-02-16T18:51:05.938116232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:51:05.942676219Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:51:05.942700033Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:51:05.94272221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:51:05.945753206Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:51:06.418272517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:51:06.421672224Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:51:08.071688442Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:51:08.985376778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:51:08.988741971Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:51:09.322289606Z caller=checkpoint.go:378 msg="writing series" size="228 kB"
ts=2026-02-16T18:51:11.32802191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:51:11.674349914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:51:11.678574876Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:51:12.43461362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="45 kB" avg_comp="22 kB" total_uncomp="305 kB" avg_uncomp="152 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:51:12.438126013Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:51:12.498484844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:51:12.502092433Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=debug ts=2026-02-16T18:51:13.054025536Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:51:13.055088177Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:51:13.071594594Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:51:13.071622521Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:51:13.071630484Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:51:13.07164894Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:51:13.07275546Z caller=checkpoint.go:378 msg="writing series" size="5.0 kB"
level=info ts=2026-02-16T18:51:14.068982913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:51:14.072460009Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:51:14.262379175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:51:14.265588614Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:51:15.710659728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="48 kB" avg_uncomp="24 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:51:15.714766931Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:51:15.980556918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:51:15.983938687Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:51:16.661609143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:51:16.665127645Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:51:16.822502958Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
ts=2026-02-16T18:51:17.439933522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:47064"
level=warn ts=2026-02-16T18:51:18.071571636Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:51:20.57290183Z caller=checkpoint.go:378 msg="writing series" size="2.4 kB"
level=info ts=2026-02-16T18:51:20.954993939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:51:20.958552319Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:51:22.11288327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:51:22.116278793Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:51:22.116308355Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:51:22.11632868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:51:22.118943645Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:51:23.071990262Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:51:23.143241599Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:51:23.380008201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:51:23.384746391Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:51:24.322230844Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-02-16T18:51:25.791012194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T18:51:25.792179214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:51:25.794387669Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:51:25.797840146Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:51:25.797856218Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:51:25.797877903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:51:25.803260996Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:51:26.363973628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="477 kB" avg_comp="159 kB" total_uncomp="4.9 MB" avg_uncomp="1.6 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:51:26.367767833Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:51:26.583375206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:51:26.586749044Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T18:51:28.071656491Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:51:28.072938936Z caller=checkpoint.go:378 msg="writing series" size="431 kB"
level=debug ts=2026-02-16T18:51:29.766534724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:51:30.302899045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:51:30.310123481Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:51:30.563909955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="443 B" total_uncomp="1.0 kB" avg_uncomp="344 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:51:30.567356041Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:51:31.822975204Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T18:51:32.64537129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="993 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:51:32.648965699Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T18:51:33.071558971Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:51:33.096851005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:51:33.100267675Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:51:33.376204073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:51:33.37967598Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:51:33.609527206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:51:33.613119085Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=debug ts=2026-02-16T18:51:35.572409877Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
level=debug ts=2026-02-16T18:51:35.572436757Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=13
level=warn ts=2026-02-16T18:51:38.072020123Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:51:39.323027129Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T18:51:41.325509344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:51:41.330760086Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T18:51:41.330850855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T18:51:43.071212243Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:51:43.071240715Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:51:43.071250217Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:51:43.071267739Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:51:43.072467228Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=info ts=2026-02-16T18:51:43.891079765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:51:43.89789013Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:51:45.251319057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:51:45.255785822Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T18:51:46.823111987Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T18:51:47.323812345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:51:47.32902801Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T18:51:48.07118664Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:51:49.522979234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:41172"
level=debug ts=2026-02-16T18:51:50.572308452Z caller=checkpoint.go:378 msg="writing series" size="158 kB"
level=debug ts=2026-02-16T18:51:51.625384501Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:51:53.071779602Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:51:54.323002483Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=info ts=2026-02-16T18:51:54.707521444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:51:54.711134384Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:51:56.08388635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:51:56.087304041Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:51:57.030976923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:51:57.034385511Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T18:51:58.071932443Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:51:58.072126206Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=info ts=2026-02-16T18:51:59.390205359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:51:59.39376884Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:52:00.916913597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:52:00.920844161Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:52:00.920873834Z caller=flush.go:240 component=ingester loop=10 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:52:00.920908276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:52:00.924212137Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=debug ts=2026-02-16T18:52:01.82279789Z caller=checkpoint.go:378 msg="writing series" size="900 B"
level=warn ts=2026-02-16T18:52:03.071113354Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:52:03.33491289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:52:03.338764695Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:52:03.884757005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:52:03.888504632Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:52:03.891522477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="48 kB" avg_uncomp="24 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:52:03.894816659Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:52:04.615109341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:52:04.618403197Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:52:05.207053746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="45 kB" avg_comp="22 kB" total_uncomp="305 kB" avg_uncomp="152 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:52:05.21155348Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T18:52:05.572698633Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T18:52:06.272424017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:52:06.276134549Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:52:06.313206237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="587 B" avg_comp="587 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:52:06.316491755Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:52:08.071809481Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:52:09.322428969Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T18:52:09.859473989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:52:09.862768951Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:52:09.930825639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:52:09.937522417Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:52:11.130186964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:52:11.133731035Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
ts=2026-02-16T18:52:11.333423286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:52:11.469358345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:52:11.475021934Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:52:13.053561476Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:52:13.054615027Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T18:52:13.071176084Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:52:13.071189778Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:52:13.071208534Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:52:13.071215916Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:52:13.07228746Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=info ts=2026-02-16T18:52:13.50120132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:52:13.504754336Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:52:15.009668124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:52:15.013628927Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:52:15.752961555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="477 kB" avg_comp="119 kB" total_uncomp="4.9 MB" avg_uncomp="1.2 MB" idle=1 full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:52:15.756803212Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:52:16.822652319Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=info ts=2026-02-16T18:52:16.901206323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:52:16.904767079Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:52:18.071353086Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:52:19.006433232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:52:19.01075584Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T18:52:19.384463048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:52:19.388306537Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
ts=2026-02-16T18:52:19.525664096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:56426"
level=debug ts=2026-02-16T18:52:19.889994655Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:52:20.069783726Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:52:20.071493932Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T18:52:20.572335079Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=info ts=2026-02-16T18:52:21.201211396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:52:21.206002974Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:52:23.071801232Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:52:24.322543618Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=info ts=2026-02-16T18:52:26.507754496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="449 B" total_uncomp="1.0 kB" avg_uncomp="344 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:52:26.511469012Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:52:26.511500521Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:52:26.511531874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="449 B" total_uncomp="1.0 kB" avg_uncomp="344 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:52:26.514663433Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:52:27.731553685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:52:27.735977289Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:52:28.071006246Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:52:28.072205838Z caller=checkpoint.go:378 msg="writing series" size="231 kB"
level=debug ts=2026-02-16T18:52:28.072225468Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=14
level=info ts=2026-02-16T18:52:30.362429557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:52:30.367098122Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T18:52:31.823105432Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T18:52:31.823136948Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=1
level=info ts=2026-02-16T18:52:32.560799906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:52:32.564634323Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:52:32.564652858Z caller=flush.go:240 component=ingester loop=8 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:52:32.564674002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:52:32.567688521Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=warn ts=2026-02-16T18:52:33.071550454Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:52:35.572524163Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=warn ts=2026-02-16T18:52:38.071997114Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:52:38.13907782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:52:38.143331785Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:52:38.147486873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:52:38.150616481Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:52:39.322502847Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T18:52:40.982374404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="449 B" total_uncomp="1.0 kB" avg_uncomp="344 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:52:40.986383806Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
ts=2026-02-16T18:52:41.336421051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:52:41.831028207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:52:41.836468536Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:52:43.071275274Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:52:43.071308677Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:52:43.071319432Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:52:43.071321355Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:52:43.073806441Z caller=checkpoint.go:378 msg="writing series" size="2.4 MB"
level=debug ts=2026-02-16T18:52:43.073822354Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.4 MB" series=3
level=info ts=2026-02-16T18:52:45.651583948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:52:45.655395832Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:52:46.774696434Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:52:46.822817452Z caller=checkpoint.go:378 msg="writing series" size="212 kB"
level=info ts=2026-02-16T18:52:47.253199564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="48 kB" avg_uncomp="24 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:52:47.257090536Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T18:52:48.071267347Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:52:49.25193556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:52:49.257397819Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=info ts=2026-02-16T18:52:49.735771986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:52:49.739427532Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:52:50.188699211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:52:50.192393033Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:52:50.48038853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:52:50.484463921Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:52:50.596435087Z caller=checkpoint.go:378 msg="writing series" size="28 MB"
level=debug ts=2026-02-16T18:52:50.596462367Z caller=checkpoint.go:390 msg="flushing series" totalSize="28 MB" series=2
level=info ts=2026-02-16T18:52:52.282722969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="46 kB" avg_comp="23 kB" total_uncomp="305 kB" avg_uncomp="152 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:52:52.287279256Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:52:52.313352854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:52:52.317173001Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:52:53.071926094Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:52:53.594442089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:52:53.598397867Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=debug ts=2026-02-16T18:52:54.322597357Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=info ts=2026-02-16T18:52:55.156059114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:52:55.159806964Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:52:55.662364316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:52:55.669596763Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:52:56.474175778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:52:56.47795571Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:52:58.071112068Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:52:58.072243208Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T18:52:58.426358327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:52:58.430598598Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:52:59.458351624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:52:59.461565552Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:53:00.097688402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:53:00.1013351Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:53:00.768275855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="477 kB" avg_comp="119 kB" total_uncomp="4.9 MB" avg_uncomp="1.2 MB" full=3 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:53:00.772029946Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:53:01.201299961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:53:01.204890229Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T18:53:01.822921734Z caller=checkpoint.go:378 msg="writing series" size="483 kB"
level=warn ts=2026-02-16T18:53:03.071724361Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:53:05.238915693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:53:05.242830582Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-02-16T18:53:05.573006966Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=warn ts=2026-02-16T18:53:08.07123615Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:53:09.322511716Z caller=checkpoint.go:378 msg="writing series" size="207 kB"
level=info ts=2026-02-16T18:53:09.490912078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:53:09.496309917Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:53:10.951681482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:53:10.955665083Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-02-16T18:53:11.338729757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:53:12.561641042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:53:12.566497964Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T18:53:13.053550507Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:53:13.054580755Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:53:13.071864904Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:53:13.071891221Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:53:13.071900853Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:53:13.071921455Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:53:13.072977428Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=debug ts=2026-02-16T18:53:16.8227474Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
ts=2026-02-16T18:53:17.447533064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:59666"
level=debug ts=2026-02-16T18:53:18.051306659Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:53:18.071471879Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:53:18.087629485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="449 B" total_uncomp="1.0 kB" avg_uncomp="344 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:53:18.09169168Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:53:20.572825069Z caller=checkpoint.go:378 msg="writing series" size="469 kB"
level=debug ts=2026-02-16T18:53:20.572850728Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=8
level=debug ts=2026-02-16T18:53:20.921446963Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T18:53:23.071902673Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:53:24.322904731Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T18:53:26.490978476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:53:26.494709975Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=info ts=2026-02-16T18:53:27.368193829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:53:27.371594742Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:53:27.826046192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:53:27.830480858Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T18:53:28.071836573Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:53:28.072959936Z caller=checkpoint.go:378 msg="writing series" size="98 kB"
level=info ts=2026-02-16T18:53:28.910274156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:53:28.913701734Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:53:31.823090503Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=warn ts=2026-02-16T18:53:33.071585835Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:53:34.313345899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:53:34.317015699Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:53:35.572681206Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T18:53:36.932833841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:53:36.936418603Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:53:37.43500712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.2 kB" total_uncomp="48 kB" avg_uncomp="24 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:53:37.439032458Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:53:37.439052828Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:53:37.439074925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.2 kB" total_uncomp="48 kB" avg_uncomp="24 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:53:37.442072306Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:53:38.040899631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:53:38.044625953Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T18:53:38.07172348Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:53:39.322587609Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
level=info ts=2026-02-16T18:53:40.956606166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:53:40.963584612Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:53:41.293592461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:53:41.297068252Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-02-16T18:53:41.341285536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:53:41.919184228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:53:41.92255452Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:53:43.070981249Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:53:43.071015436Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:53:43.071026838Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:53:43.071034209Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:53:43.072099976Z caller=checkpoint.go:390 msg="flushing series" totalSize="316 kB" series=5
level=info ts=2026-02-16T18:53:43.076944122Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-02-16T18:53:43.078114933Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.00613732s
level=info ts=2026-02-16T18:53:44.10187769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="47 kB" avg_comp="23 kB" total_uncomp="305 kB" avg_uncomp="152 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:53:44.107772489Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:53:44.115842573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="482 kB" avg_comp="121 kB" total_uncomp="4.9 MB" avg_uncomp="1.2 MB" full=3 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:53:44.119616349Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:53:44.169869782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="715 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:53:44.173311657Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:53:45.472144783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:53:45.475663986Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:53:46.053529455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:53:46.057021862Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:53:46.440808311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:53:46.445091122Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T18:53:46.91751454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:53:46.920777998Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:53:47.233700245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:53:47.236993643Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:53:48.071819609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:53:49.248435477Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:53:49.532008915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:48386"
level=info ts=2026-02-16T18:53:51.039822047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.2 kB" total_uncomp="48 kB" avg_uncomp="24 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:53:51.044713998Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:53:52.088880455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:53:52.092455675Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:53:52.092476298Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:53:52.092498233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:53:52.095203484Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:53:53.070978679Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:53:55.090876162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:53:55.094362444Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:53:57.232499676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:53:57.236389446Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:53:57.800930018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:53:57.806621646Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:53:58.071886659Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:53:59.499408961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:53:59.503143009Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:54:00.603645473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:54:00.607077934Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T18:54:00.607096723Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:54:00.607117315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:54:00.609871177Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T18:54:03.071031327Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:54:04.881774004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:54:04.886266644Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T18:54:06.525507449Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33800"
level=info ts=2026-02-16T18:54:07.420051377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:54:07.424322652Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:54:07.818598602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="935 B" avg_comp="935 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:54:07.822176172Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:54:08.07103616Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:54:08.966823938Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:54:11.343871257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T18:54:11.65390887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:54:11.657579786Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:54:12.753535962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:54:12.756791704Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:54:13.05391019Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:54:13.054959179Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:54:13.07126936Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:54:13.071298104Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:54:13.07130709Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:54:13.071295568Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-02-16T18:54:13.071337662Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:54:13.071542885Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=debug ts=2026-02-16T18:54:13.076772257Z caller=checkpoint.go:378 msg="writing series" size="7.5 MB"
level=debug ts=2026-02-16T18:54:13.076785692Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.5 MB" series=1
level=debug ts=2026-02-16T18:54:16.3422809Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:54:16.823016275Z caller=checkpoint.go:378 msg="writing series" size="439 kB"
level=info ts=2026-02-16T18:54:17.345265178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="449 B" total_uncomp="1.0 kB" avg_uncomp="344 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:54:17.349708193Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:54:18.071857773Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:54:19.105112462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.2 kB" total_uncomp="48 kB" avg_uncomp="24 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:54:19.1091723Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
ts=2026-02-16T18:54:19.534826544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:38906"
level=debug ts=2026-02-16T18:54:20.572694611Z caller=checkpoint.go:378 msg="writing series" size="159 kB"
level=info ts=2026-02-16T18:54:21.617221788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:54:21.622402175Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T18:54:23.0718583Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:54:23.206122785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:54:23.210120334Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:54:24.322233987Z caller=checkpoint.go:378 msg="writing series" size="149 kB"
level=info ts=2026-02-16T18:54:24.76202258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="47 kB" avg_comp="23 kB" total_uncomp="305 kB" avg_uncomp="152 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:54:24.765545758Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:54:25.247420053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:54:25.250927703Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:54:25.250951413Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:54:25.250978492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:54:25.253877257Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:54:25.322128639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:54:25.325556743Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=info ts=2026-02-16T18:54:26.841739765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:54:26.845448396Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T18:54:27.910816386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:54:27.916926571Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:54:28.071717979Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:54:28.071781801Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=info ts=2026-02-16T18:54:29.890424907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:54:29.894215319Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:54:30.01245257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:54:30.015866997Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:54:30.524293822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:54:30.527943552Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:54:31.655735879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:54:31.659285298Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:54:31.659306975Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:54:31.659329021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:54:31.662107945Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:54:31.822421237Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=warn ts=2026-02-16T18:54:33.071658692Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:54:34.160480038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:54:34.163993841Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:54:35.126109757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:54:35.130382209Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T18:54:35.130402249Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:54:35.130426827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:54:35.133485742Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T18:54:35.572712949Z caller=checkpoint.go:378 msg="writing series" size="897 B"
level=info ts=2026-02-16T18:54:35.985938036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:54:35.989423145Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:54:36.624128355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:54:36.627987591Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:54:36.790531853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="512 B" avg_comp="256 B" total_uncomp="545 B" avg_uncomp="272 B" idle=1 full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:54:36.79432844Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:54:37.641495027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:54:37.645025468Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:54:38.071403528Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:54:38.446560415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:54:38.450040875Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T18:54:38.450061959Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:54:38.450084403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:54:38.453288932Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T18:54:39.32273669Z caller=checkpoint.go:378 msg="writing series" size="208 kB"
level=debug ts=2026-02-16T18:54:39.322762633Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-02-16T18:54:40.043503354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="482 kB" avg_comp="121 kB" total_uncomp="4.9 MB" avg_uncomp="1.2 MB" full=3 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:54:40.047514629Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:54:40.047533687Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:54:40.047558971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="482 kB" avg_comp="121 kB" total_uncomp="4.9 MB" avg_uncomp="1.2 MB" full=3 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:54:40.051058286Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:54:41.22339629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:54:41.22690155Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T18:54:41.346286881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:54:42.654366643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:54:42.657577589Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:54:43.001579181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:54:43.004923702Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:54:43.071451118Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:54:43.071480382Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:54:43.071489284Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:54:43.071458547Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:54:43.072541464Z caller=checkpoint.go:378 msg="writing series" size="917 B"
level=debug ts=2026-02-16T18:54:45.986127425Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:54:46.822800905Z caller=checkpoint.go:378 msg="writing series" size="481 kB"
ts=2026-02-16T18:54:47.452896044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:60820"
level=warn ts=2026-02-16T18:54:48.071361143Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:54:48.312435381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:54:48.319751444Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:54:48.319782289Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:54:48.319810126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:54:48.324439636Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:54:49.472444655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:54:49.475972116Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:54:50.572293547Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
level=info ts=2026-02-16T18:54:51.048705633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="482 kB" avg_comp="121 kB" total_uncomp="4.9 MB" avg_uncomp="1.2 MB" full=3 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:54:51.052634149Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:54:53.071698907Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:54:54.322837036Z caller=checkpoint.go:378 msg="writing series" size="70 kB"
level=info ts=2026-02-16T18:54:54.473415734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:54:54.478242708Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:54:56.532420077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:54:56.536064989Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:54:57.949712537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:54:57.953627344Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T18:54:58.071330187Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:54:58.072449243Z caller=checkpoint.go:378 msg="writing series" size="154 kB"
level=info ts=2026-02-16T18:55:00.766634633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" idle=1 full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:55:00.770577828Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:55:00.770604066Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:55:00.770626734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:55:00.773358151Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:55:01.124199534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:55:01.129300607Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:55:01.823454827Z caller=checkpoint.go:378 msg="writing series" size="976 kB"
level=debug ts=2026-02-16T18:55:01.823482402Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=6
level=warn ts=2026-02-16T18:55:03.070984588Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:55:04.719221907Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:55:05.573506647Z caller=checkpoint.go:378 msg="writing series" size="605 kB"
level=info ts=2026-02-16T18:55:05.884808799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" idle=1 full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:55:05.88840889Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T18:55:08.071460277Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:55:08.901345217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.0 MB" avg_comp="2.5 MB" total_uncomp="43 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:55:08.907977207Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:55:09.322250326Z caller=checkpoint.go:378 msg="writing series" size="815 kB"
level=debug ts=2026-02-16T18:55:09.322278769Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-02-16T18:55:09.653737755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:55:09.657356386Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:55:09.657378235Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:55:09.657403533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:55:09.660445639Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:55:11.088546837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:55:11.092507797Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
ts=2026-02-16T18:55:11.349315293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=debug ts=2026-02-16T18:55:13.053552254Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:55:13.054614518Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:55:13.071798329Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:55:13.071821763Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:55:13.071830583Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:55:13.071848602Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:55:13.07217614Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB"
level=info ts=2026-02-16T18:55:13.384025398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:55:13.387980927Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:55:13.391010675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:55:13.394299305Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:55:13.571049247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:55:13.574613181Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:55:15.994923246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="449 B" total_uncomp="1.0 kB" avg_uncomp="344 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:55:15.998384907Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:55:16.344404776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="47 kB" avg_comp="23 kB" total_uncomp="305 kB" avg_uncomp="152 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:55:16.34819472Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:55:16.348219163Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:55:16.348243287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="47 kB" avg_comp="23 kB" total_uncomp="305 kB" avg_uncomp="152 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:55:16.351144507Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T18:55:16.822377554Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=debug ts=2026-02-16T18:55:16.822410778Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=2
level=debug ts=2026-02-16T18:55:17.554633808Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:55:17.680028005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:55:17.683594004Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:55:17.691661982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:55:17.695444756Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:55:17.697467651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:55:17.700649172Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:55:17.700665057Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:55:17.700684894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:55:17.703613566Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:55:18.071686758Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:55:18.535047408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:55:18.540390139Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:55:18.540413506Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:55:18.540438339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:55:18.544382639Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T18:55:18.678810817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:55:18.682416624Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=info ts=2026-02-16T18:55:18.961205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:55:18.965137367Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T18:55:20.138547991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:55:20.141808369Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:55:20.207200781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:55:20.210590344Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T18:55:20.572970807Z caller=checkpoint.go:378 msg="writing series" size="582 kB"
level=info ts=2026-02-16T18:55:20.849653026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:55:20.853292925Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T18:55:23.07171716Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:55:23.807762947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:55:23.811361647Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:55:24.322465843Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T18:55:26.07891236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="482 kB" avg_comp="121 kB" total_uncomp="4.9 MB" avg_uncomp="1.2 MB" full=3 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:55:26.083065667Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:55:26.866318844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="530 B" avg_comp="265 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:55:26.86984494Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:55:26.869867519Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:55:26.869890937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="530 B" avg_comp="265 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:55:26.872554687Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:55:28.071600486Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:55:28.071756606Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T18:55:28.37656754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:55:28.380037574Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:55:30.271579205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:55:30.276623366Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:55:31.827545784Z caller=checkpoint.go:378 msg="writing series" size="8.1 MB"
level=debug ts=2026-02-16T18:55:31.827570683Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.7 MB" series=4
level=info ts=2026-02-16T18:55:32.343014789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:55:32.346375035Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:55:32.346392109Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:55:32.346410059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:55:32.3495947Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:55:32.463051781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:55:32.46649204Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:55:32.608699958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:55:32.613279939Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T18:55:33.071951042Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:55:33.138130212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="47 kB" avg_comp="23 kB" total_uncomp="305 kB" avg_uncomp="152 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:55:33.141272975Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:55:34.066189565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:55:34.069362488Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:55:34.269642433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:55:34.273025319Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:55:34.46201612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:55:34.465073923Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:55:34.794083636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:55:34.797373573Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:55:34.797393307Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:55:34.797415466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:55:34.800286717Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:55:35.572558463Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB"
level=warn ts=2026-02-16T18:55:38.071437508Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:55:39.322620752Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
ts=2026-02-16T18:55:41.351822008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:55:43.071118356Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:55:43.071135664Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:55:43.07117767Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:55:43.071188784Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:55:43.072296948Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=debug ts=2026-02-16T18:55:44.601027628Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:55:44.984311206Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:55:45.995984857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="530 B" avg_comp="265 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:55:46.001063762Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:55:46.822791446Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T18:55:46.822816078Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-02-16T18:55:47.91112166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:55:47.914737087Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T18:55:48.07142279Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:55:49.540547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:43070"
level=debug ts=2026-02-16T18:55:50.571748219Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T18:55:51.954509554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:55:51.96016441Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:55:53.071189194Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:55:54.038650436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:55:54.04226878Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:55:54.042288892Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:55:54.042309756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:55:54.04509577Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:55:54.322752999Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T18:55:54.422009075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:55:54.425321541Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:55:55.009324505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:55:55.016105222Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:55:55.141507248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:55:55.14535128Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:55:55.145376995Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:55:55.446085805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:55:55.449952867Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:55:55.924365026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:55:55.927812033Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:55:56.02807437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="368 B" total_uncomp="1.4 kB" avg_uncomp="345 B" idle=1 full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:55:56.031348907Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:55:58.071525408Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:55:58.072699561Z caller=checkpoint.go:378 msg="writing series" size="264 kB"
level=info ts=2026-02-16T18:55:59.405282766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:55:59.41040091Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:56:00.656737852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:56:00.659961905Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:56:00.659982015Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:56:00.660004783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:56:00.662734146Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:56:01.64740404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:56:01.651836307Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T18:56:01.822330152Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
level=warn ts=2026-02-16T18:56:03.071780633Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:56:04.607211247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:56:04.613653121Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:56:05.572611499Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=info ts=2026-02-16T18:56:06.632561744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="482 kB" avg_comp="121 kB" total_uncomp="4.9 MB" avg_uncomp="1.2 MB" full=3 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:56:06.636532679Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:56:06.97045224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="47 kB" avg_comp="23 kB" total_uncomp="305 kB" avg_uncomp="152 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:56:06.973869476Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:56:07.918368351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:56:07.921903744Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:56:08.071673305Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:56:09.322599428Z caller=checkpoint.go:378 msg="writing series" size="228 kB"
level=info ts=2026-02-16T18:56:09.4017268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:56:09.405200431Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:56:10.991975585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:56:10.995581242Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=info ts=2026-02-16T18:56:11.11604907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:56:11.119390148Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T18:56:11.296021528Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:56:11.35424163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T18:56:11.992312689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:56:11.995726421Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:56:13.054210938Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:56:13.055269906Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:56:13.071454176Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:56:13.071475136Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:56:13.07148296Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:56:13.071482398Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:56:13.07260977Z caller=checkpoint.go:378 msg="writing series" size="5.0 kB"
level=info ts=2026-02-16T18:56:13.616051023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:56:13.620050207Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:56:14.189651631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:56:14.193306906Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=debug ts=2026-02-16T18:56:16.82212792Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T18:56:16.88357859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="993 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:56:16.887424588Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:56:17.305621297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:56:17.308885295Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-02-16T18:56:17.457815337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:40088"
level=info ts=2026-02-16T18:56:17.654713009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:56:17.657943266Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:56:18.071823378Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:56:18.465659667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="530 B" avg_comp="265 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:56:18.468888049Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
ts=2026-02-16T18:56:19.543693984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:53730"
level=debug ts=2026-02-16T18:56:20.572329008Z caller=checkpoint.go:378 msg="writing series" size="2.4 kB"
level=info ts=2026-02-16T18:56:20.905281178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:56:20.908575565Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T18:56:23.071898643Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:56:23.163177717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:56:23.167434158Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:56:24.322138827Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=info ts=2026-02-16T18:56:24.655760387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:56:24.659048667Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:56:27.356865766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.7 kB" avg_comp="3.7 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:56:27.36028398Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:56:28.071729679Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:56:28.072004675Z caller=checkpoint.go:378 msg="writing series" size="339 kB"
level=info ts=2026-02-16T18:56:28.903186072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:56:28.906610704Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:56:29.097129949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:56:29.098998541Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T18:56:29.51021114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:56:29.515529584Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:56:30.957070665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.0 kB" avg_comp="520 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:56:30.960784153Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:56:31.822605599Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-02-16T18:56:33.071193817Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:56:35.014900692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:56:35.018973289Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:56:35.572943906Z caller=checkpoint.go:378 msg="writing series" size="114 kB"
level=debug ts=2026-02-16T18:56:35.572969288Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=13
level=debug ts=2026-02-16T18:56:37.512638058Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:56:37.68229729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:56:37.685665148Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:56:37.994494599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="363 B" total_uncomp="1.4 kB" avg_uncomp="345 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:56:37.997831464Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:56:38.071162884Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:56:39.322080851Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T18:56:40.469108334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:56:40.473473414Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-02-16T18:56:41.35662662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=warn ts=2026-02-16T18:56:43.071047756Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:56:43.071068637Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:56:43.071092836Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:56:43.071101159Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:56:43.072286285Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=info ts=2026-02-16T18:56:43.485018696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:56:43.491480769Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:56:46.822321641Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T18:56:47.076210929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:56:47.080270498Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:56:47.884337809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:56:47.889023142Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T18:56:48.071717099Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:56:48.956120946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:56:48.959512433Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:56:49.27938672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:56:49.283235032Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T18:56:49.546581384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:54056"
level=info ts=2026-02-16T18:56:50.137822662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="47 kB" avg_comp="16 kB" total_uncomp="305 kB" avg_uncomp="102 kB" idle=1 full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:56:50.141392052Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:56:50.216597054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="482 kB" avg_comp="121 kB" total_uncomp="4.9 MB" avg_uncomp="1.2 MB" full=3 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:56:50.220187793Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:56:50.572349361Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
level=info ts=2026-02-16T18:56:50.89836274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:56:50.902009806Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T18:56:53.07129614Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:56:54.322026129Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=info ts=2026-02-16T18:56:55.120820465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:56:55.126236016Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:56:58.071760358Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:56:58.071869497Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=debug ts=2026-02-16T18:57:01.822759428Z caller=checkpoint.go:378 msg="writing series" size="900 B"
level=info ts=2026-02-16T18:57:01.838932209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:57:01.843113977Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T18:57:03.071885594Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:57:03.930978351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="530 B" avg_comp="265 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:57:03.934530618Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:57:03.970665217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:57:03.973780677Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=info ts=2026-02-16T18:57:04.975482885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:57:04.979105248Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T18:57:05.572758938Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T18:57:07.750599466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:57:07.756060241Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T18:57:08.045817242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:57:08.049450228Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T18:57:08.071680656Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:57:08.26855767Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:57:08.451348974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:57:08.454598471Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:57:09.322744919Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T18:57:09.840714632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:57:09.844479272Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:57:09.844500162Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:57:09.84452075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:57:09.847254224Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T18:57:11.35933659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=debug ts=2026-02-16T18:57:13.053486646Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:57:13.054534853Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:57:13.071666492Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:57:13.07166382Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:57:13.071685078Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:57:13.071692863Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:57:13.071706926Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=info ts=2026-02-16T18:57:14.946637152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:57:14.950343713Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:57:16.713000039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:57:16.716417267Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:57:16.822708895Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=info ts=2026-02-16T18:57:18.01452431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:57:18.018070231Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:57:18.07130696Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:57:18.83559374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:57:18.839405645Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T18:57:18.961927386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:57:18.965330493Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T18:57:19.54961507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:49750"
level=info ts=2026-02-16T18:57:19.684350547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:57:19.690252285Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:57:20.493339928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:57:20.496905235Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:57:20.537068576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="368 B" total_uncomp="1.4 kB" avg_uncomp="345 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:57:20.54079313Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:57:20.572053806Z caller=checkpoint.go:378 msg="writing series" size="39 kB"
level=warn ts=2026-02-16T18:57:23.070948627Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:57:23.639027918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:57:23.642491962Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:57:24.322023216Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=warn ts=2026-02-16T18:57:28.07138035Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:57:28.072531082Z caller=checkpoint.go:378 msg="writing series" size="231 kB"
level=debug ts=2026-02-16T18:57:28.072546783Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=14
level=debug ts=2026-02-16T18:57:29.733221753Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:57:29.962941493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:57:29.96634016Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:57:29.966363946Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:57:29.966384609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:57:29.969175658Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:57:30.118591817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:57:30.121744844Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:57:30.845633643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:57:30.851859764Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:57:30.85188096Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:57:30.851904379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:57:30.857331067Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:57:31.822660139Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-02-16T18:57:31.822684359Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=1
level=info ts=2026-02-16T18:57:32.806551817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.0 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:57:32.811047939Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T18:57:33.070980525Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:57:35.040494107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:57:35.044711628Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T18:57:35.572291631Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=warn ts=2026-02-16T18:57:38.071493368Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:57:39.321941339Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T18:57:40.299767692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:57:40.303808734Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T18:57:41.361904333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=debug ts=2026-02-16T18:57:41.366341463Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:57:41.923777559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="482 kB" avg_comp="96 kB" total_uncomp="4.9 MB" avg_uncomp="981 kB" idle=1 full=3 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:57:41.92747789Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:57:42.489335331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="587 B" avg_comp="587 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:57:42.492587891Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:57:43.071427935Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:57:43.071455385Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:57:43.071463998Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:57:43.07150807Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:57:43.073965647Z caller=checkpoint.go:378 msg="writing series" size="2.6 MB"
level=debug ts=2026-02-16T18:57:43.073978885Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.6 MB" series=3
level=info ts=2026-02-16T18:57:43.819679948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:57:43.822880975Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:57:45.921291911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:57:45.92530653Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=info ts=2026-02-16T18:57:46.227138764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:57:46.230508113Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:57:46.822143017Z caller=checkpoint.go:378 msg="writing series" size="223 kB"
ts=2026-02-16T18:57:47.463734811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:60484"
level=info ts=2026-02-16T18:57:47.576436324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:57:47.58246026Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:57:47.790976499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:57:47.794208329Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T18:57:48.071973126Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:57:48.273014814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="47 kB" avg_comp="16 kB" total_uncomp="305 kB" avg_uncomp="102 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:57:48.276210306Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
ts=2026-02-16T18:57:49.552623983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:41012"
level=debug ts=2026-02-16T18:57:50.59284094Z caller=checkpoint.go:378 msg="writing series" size="29 MB"
level=debug ts=2026-02-16T18:57:50.592868878Z caller=checkpoint.go:390 msg="flushing series" totalSize="29 MB" series=2
level=info ts=2026-02-16T18:57:51.12870688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:57:51.134249561Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:57:52.409506637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="530 B" avg_comp="265 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:57:52.41280308Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:57:53.071757015Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:57:54.322111948Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=info ts=2026-02-16T18:57:57.915576353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:57:57.92124056Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:57:58.071632691Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:57:58.07268976Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T18:57:59.196106674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:57:59.199675242Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:57:59.889517616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:57:59.892758357Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:58:00.148498332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:58:00.151687787Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T18:58:00.632695578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:58:00.636104027Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:58:01.260112756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:58:01.263346177Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:58:01.519918055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:58:01.525056028Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:58:01.821935465Z caller=checkpoint.go:378 msg="writing series" size="483 kB"
level=info ts=2026-02-16T18:58:02.242033869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:58:02.245587686Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T18:58:03.071907546Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:58:03.402946949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:58:03.406786511Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:58:03.406808621Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:58:03.406829224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:58:03.409704788Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:58:05.047419479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="368 B" total_uncomp="1.4 kB" avg_uncomp="345 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:58:05.050963996Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:58:05.169362732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:58:05.173116106Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=debug ts=2026-02-16T18:58:05.572223349Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T18:58:06.266261305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:58:06.269622198Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T18:58:08.070964954Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:58:09.322769144Z caller=checkpoint.go:378 msg="writing series" size="253 kB"
ts=2026-02-16T18:58:11.364418369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:58:12.788476746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:58:12.79350945Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:58:13.053366608Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:58:13.054409065Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:58:13.071698367Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:58:13.071716945Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:58:13.071724843Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:58:13.071747092Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:58:13.071783328Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=info ts=2026-02-16T18:58:15.567440345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:58:15.571330042Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:58:16.822049761Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
level=debug ts=2026-02-16T18:58:17.083447706Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:58:17.386263748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:58:17.392545014Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T18:58:17.466294577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46084"
level=warn ts=2026-02-16T18:58:18.07133527Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:58:18.778261114Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:58:20.572726678Z caller=checkpoint.go:378 msg="writing series" size="469 kB"
level=debug ts=2026-02-16T18:58:20.572753256Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=8
level=info ts=2026-02-16T18:58:22.75898755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:58:22.762461845Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T18:58:23.071412812Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:58:24.322734173Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T18:58:25.478109971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:58:25.481688868Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:58:26.157573954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:58:26.161738023Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T18:58:26.612954579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=3 total_comp="4.1 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="5.5 kB" idle=1 full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:58:26.616280989Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T18:58:28.071478115Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:58:28.072607415Z caller=checkpoint.go:378 msg="writing series" size="109 kB"
level=info ts=2026-02-16T18:58:30.709802348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:58:30.716813301Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:58:31.822781761Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=warn ts=2026-02-16T18:58:33.070976139Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:58:34.609280995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:58:34.61292173Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:58:34.620917865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="47 kB" avg_comp="16 kB" total_uncomp="305 kB" avg_uncomp="102 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:58:34.623947806Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:58:34.851675614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:58:34.855366234Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:58:35.070105099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="530 B" avg_comp="265 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:58:35.073345801Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:58:35.572685351Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
ts=2026-02-16T18:58:36.540238383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58798"
level=info ts=2026-02-16T18:58:37.05949787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="482 kB" avg_comp="96 kB" total_uncomp="4.9 MB" avg_uncomp="981 kB" full=4 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:58:37.063190928Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:58:37.479388659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:58:37.482644352Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T18:58:38.071251924Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:58:39.197721019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:58:39.200984818Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:58:39.321801563Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=info ts=2026-02-16T18:58:40.99020019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:58:40.993709113Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
ts=2026-02-16T18:58:41.366987874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:58:42.279864867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:58:42.283691455Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:58:42.64659207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:58:42.652035404Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:58:43.071553316Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:58:43.071578979Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:58:43.071587752Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:58:43.071602935Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:58:43.072619359Z caller=checkpoint.go:390 msg="flushing series" totalSize="328 kB" series=5
level=info ts=2026-02-16T18:58:43.078497958Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-02-16T18:58:43.081464701Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.009798637s
level=info ts=2026-02-16T18:58:43.630540507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:58:43.634410271Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:58:47.403723142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:58:47.40722948Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T18:58:48.071720845Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:58:49.041983887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:58:49.045195125Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:58:50.302797921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="368 B" total_uncomp="1.4 kB" avg_uncomp="345 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:58:50.306131551Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:58:50.306178464Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:58:50.306199102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="368 B" total_uncomp="1.4 kB" avg_uncomp="345 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:58:50.308790014Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:58:52.177875579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:58:52.181090956Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T18:58:52.332235611Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:58:52.468752983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:58:52.471918601Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:58:53.071506084Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:58:54.665595175Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:58:54.802873431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:58:54.806430817Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:58:55.019126794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:58:55.022562593Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:58:56.809204765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="715 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:58:56.812906438Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T18:58:58.071816393Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:58:59.178094645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:58:59.181744681Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:59:01.31276589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:59:01.318741346Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:59:01.318762754Z caller=flush.go:240 component=ingester loop=10 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:59:01.318784864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:59:01.321928262Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=warn ts=2026-02-16T18:59:03.071898442Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:59:04.780600719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:59:04.784807706Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:59:05.707349459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="368 B" total_uncomp="1.4 kB" avg_uncomp="345 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:59:05.710626084Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:59:08.071311762Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:59:08.986585427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:59:08.989920292Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:59:10.307314433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:59:10.313533519Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T18:59:11.369549204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T18:59:11.900366073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:59:11.903777617Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:59:12.389187078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:59:12.39277908Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=debug ts=2026-02-16T18:59:13.053894433Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T18:59:13.054939918Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:59:13.071046361Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T18:59:13.071058963Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:59:13.071081561Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:59:13.071092586Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:59:13.071093691Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:59:13.071217401Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=debug ts=2026-02-16T18:59:13.075544433Z caller=checkpoint.go:378 msg="writing series" size="8.3 MB"
level=debug ts=2026-02-16T18:59:13.075556598Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.3 MB" series=1
level=debug ts=2026-02-16T18:59:16.821970558Z caller=checkpoint.go:378 msg="writing series" size="543 kB"
level=warn ts=2026-02-16T18:59:18.071785216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:59:18.213120356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="482 kB" avg_comp="96 kB" total_uncomp="4.9 MB" avg_uncomp="981 kB" full=4 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:59:18.218595327Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:59:19.015755044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=3 total_comp="4.1 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="5.5 kB" full=3 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:59:19.019366508Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:59:20.571827601Z caller=checkpoint.go:378 msg="writing series" size="159 kB"
level=info ts=2026-02-16T18:59:21.337968271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:59:21.341257986Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:59:21.43050937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="47 kB" avg_comp="16 kB" total_uncomp="305 kB" avg_uncomp="102 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:59:21.433947899Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:59:21.605452464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:59:21.608963829Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T18:59:23.071023746Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:59:24.141398202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:59:24.145945358Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T18:59:24.32239759Z caller=checkpoint.go:378 msg="writing series" size="165 kB"
level=info ts=2026-02-16T18:59:25.38894513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:59:25.392302Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:59:26.395130847Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:59:28.071817619Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=warn ts=2026-02-16T18:59:28.071855429Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:59:28.148099864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="530 B" avg_comp="265 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:59:28.151644323Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:59:28.979026588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:59:28.983884793Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:59:29.665041904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:59:29.668765171Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:59:29.668786455Z caller=flush.go:240 component=ingester loop=8 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:59:29.668807538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:59:29.671791329Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=info ts=2026-02-16T18:59:29.9466038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:59:29.949868265Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:59:31.120350853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:59:31.124224975Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:59:31.719789091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T18:59:31.724224575Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T18:59:31.821446838Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=warn ts=2026-02-16T18:59:33.071180173Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:59:34.63272319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:59:34.636014629Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:59:35.571788517Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T18:59:36.201578613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="368 B" total_uncomp="1.4 kB" avg_uncomp="345 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:59:36.205037084Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
ts=2026-02-16T18:59:36.544387113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46084"
level=warn ts=2026-02-16T18:59:38.071078338Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:59:38.84474811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:59:38.84813632Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:59:39.259244159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:59:39.262301524Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:59:39.303958552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:59:39.306881726Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:59:39.322061357Z caller=checkpoint.go:378 msg="writing series" size="206 kB"
level=debug ts=2026-02-16T18:59:39.322076517Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=7
level=debug ts=2026-02-16T18:59:39.675418803Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:59:41.372744487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T18:59:42.46490264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T18:59:42.468410774Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T18:59:43.071329087Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:59:43.071354439Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:59:43.071363094Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:59:43.071379695Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:59:43.071409529Z caller=checkpoint.go:378 msg="writing series" size="917 B"
level=info ts=2026-02-16T18:59:44.557201035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T18:59:44.560614245Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T18:59:46.821845873Z caller=checkpoint.go:378 msg="writing series" size="378 kB"
level=info ts=2026-02-16T18:59:47.057619081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:59:47.061016706Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:59:47.202358964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T18:59:47.206083516Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T18:59:47.380663789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T18:59:47.38432532Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=warn ts=2026-02-16T18:59:48.071259587Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:59:50.180030785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:59:50.184282101Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:59:50.256487297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:59:50.259844534Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:59:50.571833794Z caller=checkpoint.go:378 msg="writing series" size="306 kB"
level=debug ts=2026-02-16T18:59:53.031238485Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:59:53.071721281Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:59:54.321507069Z caller=checkpoint.go:378 msg="writing series" size="113 kB"
level=info ts=2026-02-16T18:59:57.431059981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:59:57.434902333Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:59:58.071129168Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:59:58.072257042Z caller=checkpoint.go:378 msg="writing series" size="154 kB"
level=info ts=2026-02-16T19:00:00.000283989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=3 total_comp="4.1 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="5.5 kB" full=3 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T19:00:00.003855829Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T19:00:01.822537597Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T19:00:01.822563731Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=6
level=info ts=2026-02-16T19:00:02.126122506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="47 kB" avg_comp="16 kB" total_uncomp="305 kB" avg_uncomp="102 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:00:02.132064751Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T19:00:03.071197478Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:00:05.251930977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:00:05.258293204Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T19:00:05.572184825Z caller=checkpoint.go:378 msg="writing series" size="432 kB"
level=info ts=2026-02-16T19:00:06.164144781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T19:00:06.168124302Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T19:00:07.848742002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:00:07.852780497Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T19:00:08.071607361Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:00:08.462841035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="646 B" avg_comp="323 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:00:08.466082249Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T19:00:09.322499427Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB"
level=debug ts=2026-02-16T19:00:09.32252398Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=2
level=info ts=2026-02-16T19:00:11.194474206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="482 kB" avg_comp="96 kB" total_uncomp="4.9 MB" avg_uncomp="981 kB" full=4 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:00:11.198600169Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
ts=2026-02-16T19:00:11.375232335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T19:00:11.467759597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:00:11.47108213Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:00:11.47110534Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:00:11.471130954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:00:11.474096808Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T19:00:13.054089042Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T19:00:13.055164985Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T19:00:13.05539492Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-02-16T19:00:13.055434958Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1968075 n=1
level=debug ts=2026-02-16T19:00:13.055478269Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1968075 n=1
level=info ts=2026-02-16T19:00:13.071463905Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:00:13.071496594Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:00:13.07150543Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T19:00:13.071502311Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:00:13.07187325Z caller=checkpoint.go:378 msg="writing series" size="919 kB"
level=info ts=2026-02-16T19:00:14.600479396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:00:14.603794499Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T19:00:16.821522378Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T19:00:17.360200919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:00:17.363852189Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:00:17.363874365Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:00:17.363896997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.3 kB" avg_comp="4.3 kB" total_uncomp="17 kB" avg_uncomp="17 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T19:00:17.366734704Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-02-16T19:00:18.071209381Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T19:00:19.561033856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:49476"
level=debug ts=2026-02-16T19:00:20.571903185Z caller=checkpoint.go:378 msg="writing series" size="434 kB"
level=debug ts=2026-02-16T19:00:20.571930712Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-02-16T19:00:20.862795574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:00:20.86764249Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:00:20.867664705Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:00:20.867690014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:00:20.870596711Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T19:00:21.15854016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="368 B" total_uncomp="1.4 kB" avg_uncomp="345 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:00:21.162029884Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T19:00:22.922466389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:00:22.926454856Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T19:00:23.071128663Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:00:23.866524409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:00:23.870677734Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T19:00:24.123281708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:00:24.126517661Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T19:00:24.321450147Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T19:00:26.301581809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T19:00:26.305472839Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=info ts=2026-02-16T19:00:27.526805664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T19:00:27.530275197Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T19:00:28.07141036Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:00:28.07146045Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T19:00:28.43640668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:00:28.441486158Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T19:00:28.733177795Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:00:29.800029759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:00:29.803380503Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T19:00:30.975729654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:00:30.97913897Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T19:00:31.732802264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T19:00:31.736439232Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=debug ts=2026-02-16T19:00:31.828574456Z caller=checkpoint.go:378 msg="writing series" size="8.1 MB"
level=debug ts=2026-02-16T19:00:31.828606826Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.1 MB" series=3
level=info ts=2026-02-16T19:00:31.842893523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T19:00:31.846213764Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T19:00:31.929647969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:00:31.932790218Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T19:00:32.933253834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=2 total_comp="4.5 kB" avg_comp="2.3 kB" total_uncomp="23 kB" avg_uncomp="11 kB" idle=1 full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T19:00:32.936590077Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-02-16T19:00:33.071724326Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:00:33.244211576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:00:33.247430217Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T19:00:33.561914144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:00:33.565395515Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T19:00:33.774267457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:00:33.777376584Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T19:00:33.989196757Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T19:00:33.990805545Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T19:00:34.82916828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:00:34.83261347Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T19:00:35.571887529Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB"
level=warn ts=2026-02-16T19:00:38.071774787Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:00:39.321935347Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
ts=2026-02-16T19:00:41.3780578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T19:00:42.347452204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T19:00:42.351291327Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T19:00:43.071499727Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:00:43.071506634Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:00:43.071529557Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-02-16T19:00:43.071527118Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=info ts=2026-02-16T19:00:43.071537848Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T19:00:44.30666097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:00:44.310398731Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T19:00:44.575097416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="47 kB" avg_comp="16 kB" total_uncomp="305 kB" avg_uncomp="102 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:00:44.578977429Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T19:00:45.17671418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=3 total_comp="4.1 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="5.5 kB" full=3 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T19:00:45.179919878Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T19:00:46.823088139Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T19:00:46.823118711Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-02-16T19:00:47.22085184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:00:47.227306368Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T19:00:47.475259986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:45190"
level=warn ts=2026-02-16T19:00:48.07132891Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:00:50.572286854Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=warn ts=2026-02-16T19:00:53.071225133Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:00:53.779198122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=2 total_comp="4.5 kB" avg_comp="2.3 kB" total_uncomp="23 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T19:00:53.783079736Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-02-16T19:00:54.321832005Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T19:00:57.427135833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="664 B" avg_comp="332 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:00:57.431598052Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T19:00:58.071637633Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:00:58.071647244Z caller=checkpoint.go:378 msg="writing series" size="90 kB"
level=info ts=2026-02-16T19:01:00.766098854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="482 kB" avg_comp="96 kB" total_uncomp="4.9 MB" avg_uncomp="981 kB" full=4 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:01:00.770499245Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:01:00.770521517Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:01:00.770544589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="482 kB" avg_comp="96 kB" total_uncomp="4.9 MB" avg_uncomp="981 kB" full=4 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:01:00.774022281Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T19:01:01.821961794Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
level=info ts=2026-02-16T19:01:02.363599066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:01:02.367406238Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T19:01:03.071452338Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:01:03.65111481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T19:01:03.656334973Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T19:01:03.656356147Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:01:03.656379544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T19:01:03.659279203Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T19:01:04.044585884Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:01:05.427709215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:01:05.431328329Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T19:01:05.571923761Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-02-16T19:01:06.036900709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:01:06.041923246Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T19:01:06.444051814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T19:01:06.447434349Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=info ts=2026-02-16T19:01:06.681033015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:01:06.684436745Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T19:01:08.071260479Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:01:09.322403427Z caller=checkpoint.go:378 msg="writing series" size="228 kB"
level=info ts=2026-02-16T19:01:09.819208463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:01:09.822581464Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T19:01:11.380971158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T19:01:12.106645621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T19:01:12.110185417Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T19:01:13.053999551Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T19:01:13.055052411Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T19:01:13.071540347Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:01:13.071564865Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:01:13.071573175Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T19:01:13.071575054Z caller=checkpoint.go:378 msg="writing series" size="5.0 kB"
level=warn ts=2026-02-16T19:01:13.071577262Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:01:13.756965107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:01:13.760407414Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T19:01:14.647677566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:01:14.651740863Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T19:01:16.005642586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="993 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T19:01:16.009619134Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T19:01:16.275673611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="368 B" total_uncomp="1.4 kB" avg_uncomp="345 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:01:16.279135389Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T19:01:16.821438269Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T19:01:17.289393845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.0 kB" avg_comp="520 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:01:17.293065523Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:01:17.293086311Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:01:17.293108456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:01:17.29603069Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T19:01:18.071117489Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:01:20.571751735Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=info ts=2026-02-16T19:01:20.616856722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="482 kB" avg_comp="96 kB" total_uncomp="4.9 MB" avg_uncomp="981 kB" full=4 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:01:20.620525979Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T19:01:21.164013434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:01:21.167183548Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:01:21.167206368Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:01:21.167227677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:01:21.174523047Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T19:01:22.942956429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:01:22.946504007Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T19:01:23.070963515Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:01:23.52213677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T19:01:23.525333922Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T19:01:23.820108748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T19:01:23.824880825Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=debug ts=2026-02-16T19:01:24.32217155Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=debug ts=2026-02-16T19:01:25.489302279Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T19:01:27.191133209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:01:27.194365787Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:01:27.194385046Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:01:27.194408493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:01:27.197061565Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T19:01:27.926793704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:01:27.93229839Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:01:27.932319954Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:01:27.932343249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:01:27.936833025Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T19:01:28.071528633Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:01:28.071784408Z caller=checkpoint.go:378 msg="writing series" size="464 kB"
level=info ts=2026-02-16T19:01:28.521973629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:01:28.525649213Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T19:01:29.028274369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T19:01:29.031603512Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T19:01:29.031624488Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:01:29.031645464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T19:01:29.034337075Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T19:01:29.104840856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.7 kB" avg_comp="3.7 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:01:29.10796497Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T19:01:30.164669258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="47 kB" avg_comp="16 kB" total_uncomp="305 kB" avg_uncomp="102 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:01:30.168109028Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:01:30.168131668Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:01:30.168177199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="47 kB" avg_comp="16 kB" total_uncomp="305 kB" avg_uncomp="102 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:01:30.170982247Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T19:01:31.821668913Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-02-16T19:01:33.0715305Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:01:33.88850456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:01:33.896847829Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T19:01:35.572128881Z caller=checkpoint.go:378 msg="writing series" size="123 kB"
level=debug ts=2026-02-16T19:01:35.572169964Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=13
level=info ts=2026-02-16T19:01:36.53351394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:01:36.536726231Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T19:01:37.065480241Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:01:37.168706343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T19:01:37.172166243Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T19:01:37.231281804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:01:37.234461442Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T19:01:38.071319436Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:01:39.321614889Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T19:01:39.401872048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=3 total_comp="4.1 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="5.5 kB" full=3 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T19:01:39.405255862Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T19:01:39.405276429Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:01:39.405297048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=3 total_comp="4.1 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="5.5 kB" full=3 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T19:01:39.408012516Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T19:01:40.707085329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:01:40.712253764Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-02-16T19:01:41.384093314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T19:01:43.071664297Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T19:01:43.071688718Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:01:43.071698087Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:01:43.07171572Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T19:01:43.071747034Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=info ts=2026-02-16T19:01:44.80103414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="664 B" avg_comp="332 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:01:44.804817052Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:01:44.804838559Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:01:44.80486552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="664 B" avg_comp="332 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:01:44.807839367Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T19:01:45.658399144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T19:01:45.661829294Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T19:01:46.822006267Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T19:01:46.917433981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:01:46.920684082Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T19:01:47.23045035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=2 total_comp="4.5 kB" avg_comp="2.2 kB" total_uncomp="23 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T19:01:47.233857929Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T19:01:47.46319976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="47 kB" avg_comp="16 kB" total_uncomp="305 kB" avg_uncomp="102 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:01:47.46639759Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
ts=2026-02-16T19:01:47.480064363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:60798"
level=warn ts=2026-02-16T19:01:48.071197136Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:01:50.069103587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:01:50.072593931Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T19:01:50.572426578Z caller=checkpoint.go:378 msg="writing series" size="172 kB"
level=info ts=2026-02-16T19:01:51.024584675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:01:51.027765963Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T19:01:52.249349483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=3 total_comp="4.1 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="5.5 kB" full=3 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T19:01:52.252895479Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T19:01:52.856509507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T19:01:52.859949113Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-02-16T19:01:52.859966508Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:01:52.859988002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T19:01:52.862792241Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T19:01:53.071728329Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:01:53.162980629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:01:53.166269085Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T19:01:54.322224648Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=info ts=2026-02-16T19:01:56.660074902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="482 kB" avg_comp="96 kB" total_uncomp="4.9 MB" avg_uncomp="981 kB" full=4 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:01:56.66428817Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T19:01:57.313794524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T19:01:57.317566485Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=warn ts=2026-02-16T19:01:58.071776771Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:01:58.071866984Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=info ts=2026-02-16T19:01:58.538869897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:01:58.542355074Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:01:58.542375493Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:01:58.542397605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:01:58.545384759Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T19:01:58.917272499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="664 B" avg_comp="332 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:01:58.921120667Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T19:02:00.979426388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:02:00.982958679Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T19:02:01.3048718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:02:01.309993344Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T19:02:01.822373952Z caller=checkpoint.go:378 msg="writing series" size="900 B"
level=info ts=2026-02-16T19:02:02.138394477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:02:02.141918199Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T19:02:02.240174815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:02:02.243595116Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:02:02.243615832Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:02:02.243637634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:02:02.246461418Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T19:02:03.071762863Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:02:04.606202154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.8 kB" avg_comp="444 B" total_uncomp="1.4 kB" avg_uncomp="345 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:02:04.611591062Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T19:02:04.659304782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T19:02:04.662778983Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T19:02:05.247410927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T19:02:05.25075841Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-02-16T19:02:05.250781981Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:02:05.250804199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T19:02:05.253551018Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T19:02:05.571912311Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T19:02:05.982012483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:02:05.986704265Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:02:05.986728358Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:02:05.98675169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:02:05.990703611Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T19:02:08.071015819Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:02:08.390966331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T19:02:08.394743517Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T19:02:08.778719792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="47 kB" avg_comp="16 kB" total_uncomp="305 kB" avg_uncomp="102 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:02:08.782367754Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T19:02:09.321620878Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T19:02:09.79364482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="587 B" avg_comp="587 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:02:09.7969186Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:02:09.796942002Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:02:09.796961674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:02:09.799705261Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T19:02:09.857126123Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:02:10.657196563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:02:10.661086622Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T19:02:11.386732228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T19:02:12.368912264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:02:12.372231838Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T19:02:13.053356202Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T19:02:13.054422527Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T19:02:13.071622196Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:02:13.071640034Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-02-16T19:02:13.071632776Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=info ts=2026-02-16T19:02:13.071647704Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T19:02:13.071658112Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:02:13.29234194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:02:13.295582911Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T19:02:13.845698775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=3 total_comp="4.1 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="5.5 kB" full=3 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T19:02:13.849251283Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T19:02:15.145949565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:02:15.149575488Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T19:02:16.821646042Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=info ts=2026-02-16T19:02:17.078084485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T19:02:17.081263962Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T19:02:17.926215662Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T19:02:18.071560294Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:02:18.153869866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:02:18.156868641Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-02-16T19:02:19.567763238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:36422"
level=debug ts=2026-02-16T19:02:20.572266489Z caller=checkpoint.go:378 msg="writing series" size="40 kB"
level=info ts=2026-02-16T19:02:20.901200303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T19:02:20.904711451Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T19:02:21.517339506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:02:21.521828386Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T19:02:23.071918176Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:02:23.096998476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:02:23.100203789Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T19:02:24.32222439Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=info ts=2026-02-16T19:02:26.29139939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:02:26.294918065Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T19:02:28.071584468Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:02:28.071697835Z caller=checkpoint.go:378 msg="writing series" size="231 kB"
level=debug ts=2026-02-16T19:02:28.071715033Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=14
level=info ts=2026-02-16T19:02:28.774572024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T19:02:28.779506467Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T19:02:28.966020001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:02:28.973248361Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T19:02:29.517360755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="664 B" avg_comp="332 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:02:29.520636595Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T19:02:31.823099078Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-02-16T19:02:31.82312236Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=1
level=info ts=2026-02-16T19:02:32.745413417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:02:32.748848014Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T19:02:33.071902008Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:02:35.57197751Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
ts=2026-02-16T19:02:36.55342987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:52510"
level=warn ts=2026-02-16T19:02:38.071572437Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:02:39.321784056Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T19:02:40.75565811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:02:40.76198863Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T19:02:40.804055268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T19:02:40.807308799Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
ts=2026-02-16T19:02:41.389967555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T19:02:41.718577421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=2 total_comp="4.5 kB" avg_comp="2.3 kB" total_uncomp="23 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T19:02:41.722016572Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T19:02:43.070912174Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:02:43.070948695Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:02:43.070963415Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T19:02:43.070967185Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:02:43.07339173Z caller=checkpoint.go:378 msg="writing series" size="2.6 MB"
level=debug ts=2026-02-16T19:02:43.073405778Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.6 MB" series=3
level=debug ts=2026-02-16T19:02:44.204361374Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:02:44.396799643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:02:44.400122972Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T19:02:46.182514264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:02:46.187040194Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T19:02:46.821900351Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
ts=2026-02-16T19:02:47.484226022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:50182"
level=warn ts=2026-02-16T19:02:48.071347101Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:02:49.191921616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:02:49.195303195Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T19:02:50.332538666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:02:50.335690633Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T19:02:50.588345027Z caller=checkpoint.go:378 msg="writing series" size="29 MB"
level=debug ts=2026-02-16T19:02:50.588372533Z caller=checkpoint.go:390 msg="flushing series" totalSize="29 MB" series=2
level=info ts=2026-02-16T19:02:52.623990792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="482 kB" avg_comp="96 kB" total_uncomp="4.9 MB" avg_uncomp="981 kB" full=4 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:02:52.628015301Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T19:02:52.900788016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:02:52.904057821Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T19:02:52.938140514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:02:52.943134402Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T19:02:53.071938943Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:02:53.333762514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T19:02:53.337316331Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=debug ts=2026-02-16T19:02:54.321857224Z caller=checkpoint.go:378 msg="writing series" size="5.5 kB"
level=info ts=2026-02-16T19:02:54.47659854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:02:54.480076672Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T19:02:54.493134209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:02:54.49603058Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T19:02:54.747813396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.8 kB" avg_comp="448 B" total_uncomp="1.4 kB" avg_uncomp="345 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:02:54.75106187Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T19:02:56.006501739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T19:02:56.009774262Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T19:02:56.025835948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T19:02:56.028861869Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=info ts=2026-02-16T19:02:56.773065483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:02:56.776290307Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T19:02:58.071605235Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-02-16T19:02:58.071633352Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:02:58.346501184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="47 kB" avg_comp="12 kB" total_uncomp="306 kB" avg_uncomp="76 kB" idle=1 full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:02:58.350070504Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T19:02:59.575279957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T19:02:59.578871944Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T19:03:00.767266736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:03:00.770839821Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:03:00.770862069Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:03:01.071041944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:03:01.074765987Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T19:03:01.822189785Z caller=checkpoint.go:378 msg="writing series" size="484 kB"
level=warn ts=2026-02-16T19:03:03.071573405Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:03:05.57222196Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=warn ts=2026-02-16T19:03:08.071983979Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:03:09.321852269Z caller=checkpoint.go:378 msg="writing series" size="309 kB"
level=info ts=2026-02-16T19:03:10.745978837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=3 total_comp="4.1 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="5.5 kB" full=3 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T19:03:10.749539721Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T19:03:11.238745206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:03:11.241997655Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-02-16T19:03:11.392393189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=debug ts=2026-02-16T19:03:12.486318798Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T19:03:13.054099668Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T19:03:13.05516708Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T19:03:13.071227586Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:03:13.071249017Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:03:13.071257754Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T19:03:13.071313397Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:03:13.07240142Z caller=checkpoint.go:378 msg="writing series" size="34 kB"
level=debug ts=2026-02-16T19:03:15.466958Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:03:16.250190102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:03:16.253503903Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T19:03:16.822080458Z caller=checkpoint.go:378 msg="writing series" size="168 kB"
level=warn ts=2026-02-16T19:03:18.071303588Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:03:18.111437675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="715 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:03:18.114865744Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T19:03:20.572417641Z caller=checkpoint.go:378 msg="writing series" size="469 kB"
level=debug ts=2026-02-16T19:03:20.572443709Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=8
level=info ts=2026-02-16T19:03:22.143374643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:03:22.146815194Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T19:03:23.071524316Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:03:23.615125683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:03:23.62237071Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T19:03:23.872267287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T19:03:23.875498922Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T19:03:24.321414945Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T19:03:25.024506612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="664 B" avg_comp="332 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:03:25.02807982Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T19:03:27.888338734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:03:27.892859829Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T19:03:28.071488216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:03:28.071623889Z caller=checkpoint.go:378 msg="writing series" size="118 kB"
level=debug ts=2026-02-16T19:03:31.821600224Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=info ts=2026-02-16T19:03:32.574364405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T19:03:32.579246247Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T19:03:33.071945372Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:03:34.002278085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:03:34.00692589Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T19:03:35.169720616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="483 kB" avg_comp="96 kB" total_uncomp="4.9 MB" avg_uncomp="981 kB" full=4 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:03:35.17392011Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T19:03:35.571852545Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
level=info ts=2026-02-16T19:03:35.813484782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:03:35.817107885Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T19:03:37.025392943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:03:37.028874943Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T19:03:38.07150266Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:03:38.430493405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:03:38.433984023Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:03:38.434002405Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:03:38.434022761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:03:38.436867705Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T19:03:39.321693317Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=info ts=2026-02-16T19:03:40.363343422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T19:03:40.367396042Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T19:03:40.492779113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=2 total_comp="4.5 kB" avg_comp="2.3 kB" total_uncomp="23 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T19:03:40.496231176Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T19:03:41.282341671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="47 kB" avg_comp="12 kB" total_uncomp="306 kB" avg_uncomp="76 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:03:41.288116266Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
ts=2026-02-16T19:03:41.395406875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T19:03:41.727580808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:03:41.730896421Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T19:03:43.071775839Z caller=checkpoint.go:390 msg="flushing series" totalSize="339 kB" series=5
level=info ts=2026-02-16T19:03:43.07179719Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:03:43.071820212Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:03:43.071829028Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T19:03:43.071844639Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:03:43.078416172Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-02-16T19:03:43.083215818Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.01186127s
level=info ts=2026-02-16T19:03:43.688843127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T19:03:43.692630337Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T19:03:44.264425699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:03:44.269179995Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T19:03:44.823284138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:03:44.82687885Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T19:03:44.829921886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T19:03:44.832889969Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T19:03:45.306569756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="872 kB" avg_comp="872 kB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T19:03:45.306596511Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-02-16T19:03:45.31075183Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c67a47458%3A19c67d5cd86%3Ad6d8ff59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T19:03:46.620089833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:03:46.624866264Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T19:03:48.071931198Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T19:03:49.572941042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:42982"
level=info ts=2026-02-16T19:03:49.981826733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.8 kB" avg_comp="448 B" total_uncomp="1.4 kB" avg_uncomp="345 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:03:49.985071354Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T19:03:50.298998213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:03:50.302522739Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T19:03:51.648636059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:03:51.65230645Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T19:03:53.071254841Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:03:53.31910248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=3 total_comp="4.1 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="5.5 kB" full=3 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T19:03:53.322191995Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T19:03:53.668696796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T19:03:53.6721037Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=warn ts=2026-02-16T19:03:58.071960531Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:03:58.29671711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:03:58.300035035Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T19:04:01.12440732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:04:01.128187809Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:04:01.128208625Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:04:01.12823017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:04:01.131240273Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T19:04:03.071783099Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:04:03.257393728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="873 kB" avg_comp="873 kB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:04:03.264877111Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c67a47458%3A19c67d5cd86%3Ad6d8ff59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T19:04:03.954706697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T19:04:03.958596807Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T19:04:07.9812269Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T19:04:08.071344564Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:04:10.078491664Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T19:04:11.397790504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T19:04:11.80020594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:04:11.807288444Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:04:11.807308165Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:04:11.807331188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:04:11.813146485Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T19:04:11.915450402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:04:11.918708838Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T19:04:13.053772542Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T19:04:13.054811752Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T19:04:13.070839156Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T19:04:13.070960218Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=warn ts=2026-02-16T19:04:13.070988732Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:04:13.071005078Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:04:13.071025305Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:04:13.071033251Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T19:04:13.076372111Z caller=checkpoint.go:378 msg="writing series" size="9.1 MB"
level=debug ts=2026-02-16T19:04:13.076383131Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.1 MB" series=1
level=info ts=2026-02-16T19:04:13.134195426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:04:13.138058553Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T19:04:16.486192759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:04:16.489532045Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T19:04:16.611726313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:04:16.615904674Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T19:04:16.821814631Z caller=checkpoint.go:378 msg="writing series" size="417 kB"
ts=2026-02-16T19:04:17.490936444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:33048"
level=warn ts=2026-02-16T19:04:18.071732297Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:04:20.57139695Z caller=checkpoint.go:378 msg="writing series" size="173 kB"
level=warn ts=2026-02-16T19:04:23.071473468Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:04:23.428454922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:04:23.434093088Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:04:23.434114664Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:04:23.434135353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:04:23.436902628Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T19:04:23.818543304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:04:23.821976935Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T19:04:23.878337749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T19:04:23.881272159Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T19:04:24.214122055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T19:04:24.217593761Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T19:04:24.322061992Z caller=checkpoint.go:378 msg="writing series" size="180 kB"
level=info ts=2026-02-16T19:04:24.681762748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="664 B" avg_comp="332 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:04:24.684981075Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T19:04:25.797707667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="47 kB" avg_comp="12 kB" total_uncomp="306 kB" avg_uncomp="76 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:04:25.801278914Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T19:04:26.994471021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:04:27.001219745Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T19:04:27.656700375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:04:27.660314486Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T19:04:28.071273505Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:04:28.071433462Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=info ts=2026-02-16T19:04:28.281388534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:04:28.286318316Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T19:04:28.478964425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:04:28.482630787Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T19:04:29.226688434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="873 kB" avg_comp="873 kB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:04:29.231305995Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c67a47458%3A19c67d5cd86%3Ad6d8ff59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T19:04:30.028336815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:04:30.031784861Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T19:04:31.406555495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T19:04:31.409703673Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T19:04:31.821787642Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=warn ts=2026-02-16T19:04:33.071165888Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:04:33.092276391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:04:33.095254183Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T19:04:34.263507357Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:04:35.140578239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="483 kB" avg_comp="96 kB" total_uncomp="4.9 MB" avg_uncomp="981 kB" full=4 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:04:35.145787253Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T19:04:35.371213246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.8 kB" avg_comp="448 B" total_uncomp="1.4 kB" avg_uncomp="345 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:04:35.37442292Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T19:04:35.539808112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=2 total_comp="4.5 kB" avg_comp="2.3 kB" total_uncomp="23 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T19:04:35.543007032Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-02-16T19:04:35.571194796Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=info ts=2026-02-16T19:04:35.615245989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:04:35.61823807Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-02-16T19:04:36.560464949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55250"
level=info ts=2026-02-16T19:04:37.49448837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T19:04:37.498232193Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=warn ts=2026-02-16T19:04:38.071786425Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:04:39.322424555Z caller=checkpoint.go:378 msg="writing series" size="206 kB"
level=debug ts=2026-02-16T19:04:39.322459781Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-02-16T19:04:40.139659853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:04:40.143788441Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T19:04:41.400860401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T19:04:42.343807806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=3 total_comp="4.1 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="5.5 kB" full=3 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T19:04:42.348064869Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T19:04:43.071183363Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:04:43.071215116Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:04:43.071223751Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T19:04:43.071237104Z caller=checkpoint.go:378 msg="writing series" size="917 B"
level=warn ts=2026-02-16T19:04:43.071242044Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:04:43.255731128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:04:43.259082018Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T19:04:45.476858649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T19:04:45.480433807Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=debug ts=2026-02-16T19:04:46.822410876Z caller=checkpoint.go:378 msg="writing series" size="493 kB"
ts=2026-02-16T19:04:47.4939874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:34252"
level=warn ts=2026-02-16T19:04:48.071474795Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:04:50.157005799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:04:50.160539404Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T19:04:50.571809881Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
level=warn ts=2026-02-16T19:04:53.071941501Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:04:54.321947493Z caller=checkpoint.go:378 msg="writing series" size="113 kB"
level=info ts=2026-02-16T19:04:54.444845919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T19:04:54.450190881Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T19:04:55.214373107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:04:55.221419397Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T19:04:56.252254691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:04:56.255641473Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T19:04:56.444009613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:04:56.447543545Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T19:04:58.071402652Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:04:58.071433575Z caller=checkpoint.go:378 msg="writing series" size="154 kB"
level=debug ts=2026-02-16T19:04:59.743031098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T19:04:59.744561547Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T19:05:01.822685811Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T19:05:01.822711303Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.1 MB" series=6
level=warn ts=2026-02-16T19:05:03.071193327Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:05:05.157796731Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T19:05:05.572101113Z caller=checkpoint.go:378 msg="writing series" size="447 kB"
level=info ts=2026-02-16T19:05:06.458474962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:05:06.462489983Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T19:05:06.563640066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39364"
level=warn ts=2026-02-16T19:05:08.07169094Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:05:08.43182578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:05:08.436385228Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T19:05:09.321954769Z caller=checkpoint.go:378 msg="writing series" size="986 kB"
level=debug ts=2026-02-16T19:05:09.321986584Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-02-16T19:05:10.33409782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:05:10.338063926Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T19:05:10.753192331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:05:10.756905728Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T19:05:11.151012975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:05:11.154494032Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T19:05:11.403986502Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T19:05:12.279897171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="873 kB" avg_comp="873 kB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:05:12.284644294Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c67a47458%3A19c67d5cd86%3Ad6d8ff59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T19:05:13.054216101Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T19:05:13.055279644Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T19:05:13.071088184Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:05:13.071093346Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:05:13.071111024Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:05:13.071120229Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T19:05:13.071473946Z caller=checkpoint.go:378 msg="writing series" size="897 kB"
level=info ts=2026-02-16T19:05:13.365187568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="839 B" avg_comp="279 B" total_uncomp="732 B" avg_uncomp="244 B" idle=1 full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:05:13.372788716Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T19:05:13.502114715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:05:13.505771264Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T19:05:14.566660242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:05:14.570848406Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T19:05:16.821225511Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
ts=2026-02-16T19:05:17.496823375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:59132"
level=info ts=2026-02-16T19:05:17.641407904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:05:17.646284018Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T19:05:17.733495361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=2 total_comp="4.5 kB" avg_comp="2.3 kB" total_uncomp="23 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T19:05:17.736504128Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-02-16T19:05:18.071555442Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:05:18.231852836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T19:05:18.234984836Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T19:05:18.28348974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="483 kB" avg_comp="96 kB" total_uncomp="4.9 MB" avg_uncomp="981 kB" full=4 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:05:18.286752986Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T19:05:20.572136676Z caller=checkpoint.go:378 msg="writing series" size="440 kB"
level=debug ts=2026-02-16T19:05:20.572179334Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-02-16T19:05:21.611707395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T19:05:21.615001978Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T19:05:21.678467093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:05:21.681679811Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T19:05:21.861197108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="47 kB" avg_comp="12 kB" total_uncomp="306 kB" avg_uncomp="76 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:05:21.864383194Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T19:05:23.071955993Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:05:23.319720334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T19:05:23.322996297Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T19:05:24.321411637Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T19:05:27.395597187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T19:05:27.399547512Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=warn ts=2026-02-16T19:05:28.071584987Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:05:28.071622843Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T19:05:29.092665407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="384 B" total_uncomp="1.7 kB" avg_uncomp="347 B" idle=1 full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:05:29.096425992Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:05:29.096447952Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:05:29.096469279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="384 B" total_uncomp="1.7 kB" avg_uncomp="347 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:05:29.09977547Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T19:05:29.379859052Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:05:31.336020925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:05:31.339473885Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T19:05:31.826232103Z caller=checkpoint.go:378 msg="writing series" size="8.1 MB"
level=debug ts=2026-02-16T19:05:31.826258984Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.1 MB" series=3
level=warn ts=2026-02-16T19:05:33.071061507Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:05:34.859702801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:05:34.863567534Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T19:05:35.417196834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T19:05:35.420834015Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T19:05:35.420852994Z caller=flush.go:240 component=ingester loop=10 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:05:35.420872451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T19:05:35.423960346Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=debug ts=2026-02-16T19:05:35.571691094Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB"
level=info ts=2026-02-16T19:05:36.851923047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:05:36.856863808Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T19:05:37.170220558Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T19:05:38.071489047Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:05:39.168677639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T19:05:39.172072878Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T19:05:39.322054029Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
level=info ts=2026-02-16T19:05:39.358259598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:05:39.361604665Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T19:05:39.977565736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=3 total_comp="4.1 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="5.5 kB" full=3 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T19:05:39.981054405Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T19:05:40.674656874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:05:40.678051994Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-02-16T19:05:41.406999828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T19:05:43.071231579Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:05:43.071265969Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:05:43.071277488Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T19:05:43.071310246Z caller=checkpoint.go:378 msg="writing series" size="4.0 kB"
level=warn ts=2026-02-16T19:05:43.071327432Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:05:44.738407183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:05:44.746860348Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T19:05:45.510869599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="384 B" total_uncomp="1.7 kB" avg_uncomp="347 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:05:45.51448515Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T19:05:46.823418893Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T19:05:46.823445985Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=4
ts=2026-02-16T19:05:47.499500334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:45086"
level=warn ts=2026-02-16T19:05:48.071699732Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:05:50.57199174Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T19:05:50.938028948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:05:50.941454452Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T19:05:51.658058538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T19:05:51.661678976Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=warn ts=2026-02-16T19:05:53.071370835Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:05:54.321897276Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T19:05:54.909368884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:05:54.91390102Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T19:05:54.940981839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="873 kB" avg_comp="873 kB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:05:54.945449251Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c67a47458%3A19c67d5cd86%3Ad6d8ff59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T19:05:55.614292814Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:05:56.252357252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:05:56.255849433Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T19:05:58.071681446Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:05:58.071768029Z caller=checkpoint.go:378 msg="writing series" size="141 kB"
level=info ts=2026-02-16T19:05:59.748728827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="483 kB" avg_comp="96 kB" total_uncomp="4.9 MB" avg_uncomp="981 kB" full=4 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:05:59.753849769Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T19:06:01.821860728Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
level=info ts=2026-02-16T19:06:01.887270645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="857 B" avg_comp="285 B" total_uncomp="732 B" avg_uncomp="244 B" full=3 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:06:01.890842545Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T19:06:01.985081419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T19:06:01.988960235Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T19:06:02.022037075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T19:06:02.025173602Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T19:06:03.071845111Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:06:04.543137559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:06:04.546390055Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T19:06:05.57178265Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-02-16T19:06:06.313407848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:06:06.317357694Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T19:06:06.5148165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.7 kB" avg_comp="3.7 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:06:06.518307403Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T19:06:07.773661829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:06:07.777670459Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T19:06:08.071843432Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:06:08.467748079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:06:08.474307117Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T19:06:09.023639864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:06:09.0274013Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T19:06:09.321293887Z caller=checkpoint.go:378 msg="writing series" size="228 kB"
ts=2026-02-16T19:06:11.410201188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=debug ts=2026-02-16T19:06:13.053595575Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T19:06:13.054634335Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T19:06:13.070953651Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T19:06:13.070959844Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:06:13.0709862Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:06:13.070996343Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T19:06:13.072015663Z caller=checkpoint.go:378 msg="writing series" size="5.0 kB"
level=info ts=2026-02-16T19:06:15.883362272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=2 total_comp="4.5 kB" avg_comp="2.3 kB" total_uncomp="23 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T19:06:15.887537296Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T19:06:16.022768696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="47 kB" avg_comp="12 kB" total_uncomp="306 kB" avg_uncomp="76 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:06:16.026429145Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T19:06:16.821759161Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T19:06:16.977433997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="384 B" total_uncomp="1.7 kB" avg_uncomp="347 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:06:16.981072085Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T19:06:17.238745679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T19:06:17.24217927Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T19:06:18.071215904Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:06:19.584875905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T19:06:19.589306033Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T19:06:19.870271123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:06:19.87359707Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T19:06:20.543399903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.0 kB" avg_comp="520 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:06:20.546561391Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T19:06:20.571723915Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=info ts=2026-02-16T19:06:21.166508342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:06:21.170003866Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T19:06:21.228062128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T19:06:21.231279171Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T19:06:21.231299802Z caller=flush.go:240 component=ingester loop=8 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:06:21.231321681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T19:06:21.23398093Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=warn ts=2026-02-16T19:06:23.07168969Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:06:24.321810972Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=debug ts=2026-02-16T19:06:25.789998749Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T19:06:28.071476455Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:06:28.071695228Z caller=checkpoint.go:378 msg="writing series" size="364 kB"
level=info ts=2026-02-16T19:06:28.141688709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:06:28.145520334Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T19:06:30.0990263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:06:30.103225106Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T19:06:30.801322449Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T19:06:31.822041134Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-02-16T19:06:33.071861259Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:06:33.462063638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=3 total_comp="4.1 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="5.5 kB" full=3 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T19:06:33.465432762Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T19:06:35.571684838Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
level=debug ts=2026-02-16T19:06:35.571711111Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=13
ts=2026-02-16T19:06:36.569369556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:45340"
level=info ts=2026-02-16T19:06:36.859773896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="993 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T19:06:36.863456815Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T19:06:38.032783974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T19:06:38.037981232Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=warn ts=2026-02-16T19:06:38.071432265Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:06:38.281200596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:06:38.288726698Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T19:06:39.124084584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:06:39.127740268Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T19:06:39.321393418Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
ts=2026-02-16T19:06:41.412481974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T19:06:42.853064922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T19:06:42.856622196Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T19:06:43.071383126Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:06:43.071412988Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:06:43.071420787Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T19:06:43.071408665Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:06:43.071470305Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=info ts=2026-02-16T19:06:44.817215235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:06:44.822493142Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T19:06:46.012651136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="857 B" avg_comp="285 B" total_uncomp="732 B" avg_uncomp="244 B" full=3 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:06:46.016949275Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T19:06:46.822080625Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-02-16T19:06:48.071381524Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:06:48.213686922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:06:48.2174141Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T19:06:48.355068124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:06:48.358409061Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T19:06:48.660487682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="483 kB" avg_comp="96 kB" total_uncomp="4.9 MB" avg_uncomp="981 kB" full=4 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:06:48.664412594Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T19:06:49.258644158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="873 kB" avg_comp="873 kB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:06:49.262974464Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c67a47458%3A19c67d5cd86%3Ad6d8ff59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T19:06:50.571355107Z caller=checkpoint.go:378 msg="writing series" size="199 kB"
level=info ts=2026-02-16T19:06:51.190741091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:06:51.194321293Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:06:51.194342832Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:06:51.194369336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:06:51.19701968Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T19:06:51.271562598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:06:51.277009419Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T19:06:52.137313776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T19:06:52.140738729Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T19:06:53.071433898Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:06:54.321549829Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=info ts=2026-02-16T19:06:56.116133783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=2 total_comp="4.5 kB" avg_comp="2.2 kB" total_uncomp="23 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T19:06:56.119564933Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c67a4d697%3A19c67a58626%3A8eec0947?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T19:06:56.119586289Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:06:56.119607372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:06:56.122457258Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T19:06:56.643677234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:06:56.647258666Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:06:56.647275825Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:06:56.647296119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:06:56.65007475Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T19:06:58.071491245Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:06:58.071611343Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=info ts=2026-02-16T19:06:58.122671126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T19:06:58.126294463Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T19:07:01.821212882Z caller=checkpoint.go:378 msg="writing series" size="900 B"
level=info ts=2026-02-16T19:07:02.674794442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:07:02.680733408Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T19:07:03.071515983Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:07:03.940537219Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:07:04.233760243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:07:04.237429842Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T19:07:05.157672405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:07:05.161350589Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T19:07:05.571444556Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T19:07:06.44182752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:07:06.445490096Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T19:07:08.071758351Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:07:08.239191439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:07:08.243641466Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T19:07:09.322133666Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T19:07:09.723963644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T19:07:09.727534923Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T19:07:10.198512691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T19:07:10.201922564Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=info ts=2026-02-16T19:07:10.715367306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:07:10.718717037Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T19:07:11.415859452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T19:07:11.628902511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="47 kB" avg_comp="12 kB" total_uncomp="306 kB" avg_uncomp="76 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:07:11.632144865Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T19:07:13.053886928Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T19:07:13.054953323Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T19:07:13.071680479Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:07:13.071711745Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:07:13.071720651Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T19:07:13.071731395Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=warn ts=2026-02-16T19:07:13.071777862Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:07:13.336782722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:07:13.340316968Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T19:07:14.677444517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="380 B" total_uncomp="1.7 kB" avg_uncomp="347 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:07:14.681398193Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T19:07:15.429496705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=3 total_comp="4.1 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="5.5 kB" full=3 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T19:07:15.432889023Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T19:07:16.822078243Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=info ts=2026-02-16T19:07:16.946627168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:07:16.95063504Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T19:07:18.071320122Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:07:18.580110506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:07:18.583767485Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-02-16T19:07:19.583362818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:43466"
level=info ts=2026-02-16T19:07:20.13342097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="587 B" avg_comp="587 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:07:20.136801863Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T19:07:20.571743449Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=warn ts=2026-02-16T19:07:23.071666871Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:07:23.376676384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T19:07:23.38005712Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T19:07:23.380078622Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:07:23.380103084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T19:07:23.382857194Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T19:07:24.184357776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:07:24.190688388Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T19:07:24.321602604Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=debug ts=2026-02-16T19:07:27.824913952Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T19:07:28.07099861Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:07:28.072207979Z caller=checkpoint.go:378 msg="writing series" size="231 kB"
level=debug ts=2026-02-16T19:07:28.072227786Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=14
level=info ts=2026-02-16T19:07:28.776754137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="483 kB" avg_comp="96 kB" total_uncomp="4.9 MB" avg_uncomp="981 kB" full=4 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:07:28.780505568Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:07:28.780526882Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:07:28.780550064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="483 kB" avg_comp="96 kB" total_uncomp="4.9 MB" avg_uncomp="981 kB" full=4 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:07:28.783552473Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T19:07:30.349538908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:07:30.352843379Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T19:07:30.704758509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:07:30.708112002Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T19:07:31.822478599Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-02-16T19:07:31.822503361Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=1
level=info ts=2026-02-16T19:07:32.829909785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.7 kB" avg_comp="861 B" total_uncomp="6.9 kB" avg_uncomp="3.4 kB" idle=1 full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T19:07:32.833879488Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T19:07:33.071938842Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:07:33.966908227Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:07:34.487883577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:07:34.492629628Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:07:34.492654227Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:07:34.492678279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:07:34.496837837Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T19:07:35.494080324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T19:07:35.497577916Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c67a4de0e%3A19c67a4de0f%3Ae065fcb9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T19:07:35.497600261Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:07:35.497624397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-02-16T19:07:35.500784837Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19c67a467a5%3A19c67d4c60a%3A148796e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-02-16T19:07:35.572064866Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=info ts=2026-02-16T19:07:35.727528658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:07:35.731724245Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T19:07:36.291425245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="873 kB" avg_comp="873 kB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:07:36.295706879Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c67a47458%3A19c67d5cd86%3Ad6d8ff59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-02-16T19:07:38.071601981Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:07:39.321541079Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T19:07:39.488083751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T19:07:39.494555673Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T19:07:40.975204178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:07:40.978730032Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T19:07:41.419114036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T19:07:41.451676443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:07:41.455298489Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T19:07:43.071870806Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T19:07:43.071861639Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:07:43.071897149Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:07:43.071905488Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T19:07:43.07539907Z caller=checkpoint.go:378 msg="writing series" size="2.5 MB"
level=debug ts=2026-02-16T19:07:43.07542411Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.5 MB" series=3
level=info ts=2026-02-16T19:07:43.396029352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="483 kB" avg_comp="96 kB" total_uncomp="4.9 MB" avg_uncomp="981 kB" full=4 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:07:43.400025384Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T19:07:44.771018624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:07:44.774622231Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T19:07:45.235792305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="857 B" avg_comp="285 B" total_uncomp="732 B" avg_uncomp="244 B" full=3 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:07:45.23914017Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T19:07:46.51220041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:07:46.515817883Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:07:46.515842321Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:07:46.51586643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:07:46.519167625Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T19:07:46.82210388Z caller=checkpoint.go:378 msg="writing series" size="228 kB"
level=info ts=2026-02-16T19:07:46.951636733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T19:07:46.95702359Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T19:07:47.180695267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:07:47.184301452Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T19:07:48.071604499Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:07:49.896109261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:07:49.901221739Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T19:07:50.600987985Z caller=checkpoint.go:378 msg="writing series" size="29 MB"
level=debug ts=2026-02-16T19:07:50.601019565Z caller=checkpoint.go:390 msg="flushing series" totalSize="29 MB" series=2
level=warn ts=2026-02-16T19:07:53.071302616Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:07:53.179680688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-02-16T19:07:53.184075515Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19c67a467a5%3A19c67d4c60a%3A148796e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-02-16T19:07:54.321657201Z caller=checkpoint.go:378 msg="writing series" size="5.5 kB"
level=info ts=2026-02-16T19:07:54.686368049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="47 kB" avg_comp="12 kB" total_uncomp="306 kB" avg_uncomp="76 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:07:54.69013296Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:07:54.690174757Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:07:54.690200208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="47 kB" avg_comp="12 kB" total_uncomp="306 kB" avg_uncomp="76 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:07:54.693422521Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T19:07:56.003962457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T19:07:56.007410886Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=info ts=2026-02-16T19:07:57.460677983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:07:57.464042742Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T19:07:58.071813622Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:07:58.071886167Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=debug ts=2026-02-16T19:07:58.48599997Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:07:58.565206927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:07:58.568754567Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T19:07:58.650110163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=3 total_comp="4.1 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="5.5 kB" full=3 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T19:07:58.653912652Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T19:07:58.65393804Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:07:58.653962972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=3 total_comp="4.1 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="5.5 kB" full=3 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T19:07:58.656717013Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T19:07:59.436771201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:07:59.44016749Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T19:08:01.394708818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T19:08:01.398598157Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T19:08:01.474134317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:08:01.477378561Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T19:08:01.821452273Z caller=checkpoint.go:378 msg="writing series" size="493 kB"
level=info ts=2026-02-16T19:08:02.822977925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:08:02.82657339Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:08:02.826594447Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:08:02.826615717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:08:02.829588726Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T19:08:03.071824619Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:08:05.571585861Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=warn ts=2026-02-16T19:08:08.071423386Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:08:08.136582516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="47 kB" avg_comp="12 kB" total_uncomp="306 kB" avg_uncomp="76 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:08:08.141575055Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T19:08:09.234877455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="384 B" total_uncomp="1.7 kB" avg_uncomp="347 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:08:09.238787763Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T19:08:09.321843717Z caller=checkpoint.go:378 msg="writing series" size="107 kB"
level=info ts=2026-02-16T19:08:10.436425027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="483 kB" avg_comp="96 kB" total_uncomp="4.9 MB" avg_uncomp="981 kB" full=4 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:08:10.440562888Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
ts=2026-02-16T19:08:11.422443542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T19:08:12.318828312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:08:12.322533613Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T19:08:13.053944415Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T19:08:13.055023289Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T19:08:13.071336271Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:08:13.07133783Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=info ts=2026-02-16T19:08:13.071354263Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:08:13.07136952Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:08:13.071379373Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T19:08:13.224876899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=3 total_comp="4.1 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="5.5 kB" full=3 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T19:08:13.228444909Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T19:08:14.341684592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:08:14.345734223Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T19:08:14.909843666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:08:14.917410269Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T19:08:15.929441607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:08:15.932779603Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T19:08:16.822313678Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=warn ts=2026-02-16T19:08:18.071467064Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:08:18.810216455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T19:08:18.813576803Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T19:08:19.883011138Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T19:08:20.571507311Z caller=checkpoint.go:378 msg="writing series" size="469 kB"
level=debug ts=2026-02-16T19:08:20.571531408Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-02-16T19:08:21.510112442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="873 kB" avg_comp="873 kB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:08:21.514508566Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c67a47458%3A19c67d5cd86%3Ad6d8ff59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-02-16T19:08:23.071211499Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:08:24.277127932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:08:24.280878695Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T19:08:24.321978329Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T19:08:24.328086905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.7 kB" avg_comp="861 B" total_uncomp="6.9 kB" avg_uncomp="3.4 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T19:08:24.331563615Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-02-16T19:08:24.331584341Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:08:24.331607586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.7 kB" avg_comp="861 B" total_uncomp="6.9 kB" avg_uncomp="3.4 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T19:08:24.334638762Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T19:08:24.781693702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:08:24.785416828Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T19:08:25.097330781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:08:25.101896019Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:08:25.101921139Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:08:25.10194742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:08:25.105984727Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T19:08:25.213747938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-02-16T19:08:25.217733782Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19c67a467a5%3A19c67d4c60a%3A148796e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-02-16T19:08:26.222583135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:08:26.226137536Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T19:08:26.466067053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="715 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:08:26.469512441Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:08:26.469536127Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:08:26.469563518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:08:26.472406909Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T19:08:26.677050218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:08:26.682559879Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T19:08:28.071842748Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:08:28.071942174Z caller=checkpoint.go:378 msg="writing series" size="129 kB"
level=debug ts=2026-02-16T19:08:31.822020833Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=info ts=2026-02-16T19:08:32.844861658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T19:08:32.848691818Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-02-16T19:08:32.848716144Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:08:32.848741349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T19:08:32.851706107Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T19:08:33.071606175Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:08:33.107812434Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T19:08:35.571382111Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
level=info ts=2026-02-16T19:08:35.887375843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:08:35.890907431Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:08:35.890929697Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:08:35.890953204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:08:35.893782886Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T19:08:36.575714415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39248"
level=warn ts=2026-02-16T19:08:38.071271909Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:08:38.499349816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:08:38.503610334Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T19:08:39.027991212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:08:39.032774356Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T19:08:39.322282685Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
level=info ts=2026-02-16T19:08:40.044163293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:08:40.0478957Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:08:40.047925289Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:08:40.047957953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:08:40.051294236Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T19:08:40.347265519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:08:40.350746725Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T19:08:41.056731126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:08:41.061783927Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T19:08:41.425707427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T19:08:42.052949799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="47 kB" avg_comp="12 kB" total_uncomp="306 kB" avg_uncomp="76 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:08:42.056617184Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T19:08:42.775565428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=3 total_comp="4.1 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="5.5 kB" full=3 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T19:08:42.779144275Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T19:08:43.071121681Z caller=checkpoint.go:390 msg="flushing series" totalSize="351 kB" series=5
level=info ts=2026-02-16T19:08:43.071183476Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T19:08:43.07118965Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:08:43.071207729Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:08:43.071216926Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T19:08:43.077076719Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-02-16T19:08:43.081713496Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.01062082s
level=info ts=2026-02-16T19:08:43.435120108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="857 B" avg_comp="285 B" total_uncomp="732 B" avg_uncomp="244 B" full=3 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:08:43.438405683Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:08:43.438427845Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:08:43.438461823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="857 B" avg_comp="285 B" total_uncomp="732 B" avg_uncomp="244 B" full=3 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:08:43.441068759Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T19:08:43.906281155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.7 kB" avg_comp="852 B" total_uncomp="6.9 kB" avg_uncomp="3.4 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T19:08:43.909838794Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T19:08:44.22803175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T19:08:44.231646026Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=info ts=2026-02-16T19:08:44.767052557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:08:44.770518661Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T19:08:46.584550307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:08:46.588036733Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T19:08:47.201887921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:08:47.205286273Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T19:08:47.50807719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46122"
level=warn ts=2026-02-16T19:08:48.071972195Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:08:48.89220407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:08:48.896105452Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:08:48.896127367Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:08:48.896169044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:08:48.898969122Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T19:08:50.306135267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T19:08:50.309697531Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T19:08:53.071241493Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:08:55.069851763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T19:08:55.074200735Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T19:08:55.9506167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="483 kB" avg_comp="96 kB" total_uncomp="4.9 MB" avg_uncomp="981 kB" full=4 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:08:55.955176396Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T19:08:58.071912737Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:08:58.109195505Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:08:58.436063658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:08:58.439593336Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T19:08:59.287921552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:08:59.291740095Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T19:08:59.818176855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="384 B" total_uncomp="1.7 kB" avg_uncomp="347 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:08:59.821505548Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T19:09:01.947654014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T19:09:01.949342112Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T19:09:02.845679568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="857 B" avg_comp="285 B" total_uncomp="732 B" avg_uncomp="244 B" full=3 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:09:02.84907458Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T19:09:02.893286459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:09:02.90024508Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T19:09:03.071142724Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:09:04.153267742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:09:04.156603607Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T19:09:06.578315863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50328"
level=info ts=2026-02-16T19:09:07.32593642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="873 kB" avg_comp="873 kB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:09:07.331851135Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c67a47458%3A19c67d5cd86%3Ad6d8ff59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T19:09:07.715143703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:09:07.718620619Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T19:09:08.071872045Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:09:08.764205285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:09:08.769238842Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T19:09:09.743952262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:09:09.747244795Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-02-16T19:09:11.428981065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T19:09:11.883707014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.7 kB" avg_comp="861 B" total_uncomp="6.9 kB" avg_uncomp="3.4 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T19:09:11.887621514Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T19:09:12.408032806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:09:12.4116623Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T19:09:13.025634378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:09:13.029013798Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T19:09:13.053397804Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T19:09:13.054434997Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T19:09:13.070838927Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T19:09:13.07100258Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-02-16T19:09:13.071109958Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:09:13.071144286Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:09:13.071166076Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T19:09:13.071146551Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:09:13.077507869Z caller=checkpoint.go:378 msg="writing series" size="9.4 MB"
level=debug ts=2026-02-16T19:09:13.077524565Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.4 MB" series=1
level=info ts=2026-02-16T19:09:14.062125749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T19:09:14.067059961Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T19:09:16.763375394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:09:16.767846911Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T19:09:16.822248938Z caller=checkpoint.go:378 msg="writing series" size="500 kB"
level=info ts=2026-02-16T19:09:17.794751779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:09:17.800190012Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T19:09:18.071188927Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:09:20.270360489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:09:20.274750242Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T19:09:20.57168335Z caller=checkpoint.go:378 msg="writing series" size="173 kB"
level=info ts=2026-02-16T19:09:21.589596171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-02-16T19:09:21.593939397Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19c67a467a5%3A19c67d4c60a%3A148796e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-02-16T19:09:23.071720909Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:09:24.315578275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:09:24.319845814Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T19:09:24.322052061Z caller=checkpoint.go:378 msg="writing series" size="201 kB"
level=info ts=2026-02-16T19:09:25.580138178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T19:09:25.583725374Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=info ts=2026-02-16T19:09:26.337802196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="47 kB" avg_comp="12 kB" total_uncomp="306 kB" avg_uncomp="76 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:09:26.341418583Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T19:09:26.974286408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:09:26.97756621Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T19:09:28.071654078Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:09:28.071669661Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=info ts=2026-02-16T19:09:28.321355874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T19:09:28.324585446Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T19:09:28.677558439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:09:28.680803373Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T19:09:29.399215639Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:09:29.675267111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="857 B" avg_comp="285 B" total_uncomp="732 B" avg_uncomp="244 B" full=3 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:09:29.678659737Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T19:09:31.822244379Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=warn ts=2026-02-16T19:09:33.071264186Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:09:34.021327389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:09:34.024637853Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T19:09:35.57214311Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=info ts=2026-02-16T19:09:36.482826001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:09:36.487093532Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T19:09:37.276355497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:09:37.279755727Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T19:09:38.071212876Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:09:39.321667286Z caller=checkpoint.go:378 msg="writing series" size="206 kB"
level=debug ts=2026-02-16T19:09:39.321695035Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-02-16T19:09:40.084428514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=3 total_comp="4.1 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="5.5 kB" full=3 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T19:09:40.087944427Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-csi-drivers\"}"
ts=2026-02-16T19:09:41.431739417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"
level=info ts=2026-02-16T19:09:43.071129078Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:09:43.071167567Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:09:43.071177083Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T19:09:43.071179967Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:09:43.071222718Z caller=checkpoint.go:378 msg="writing series" size="917 B"
level=debug ts=2026-02-16T19:09:46.822205406Z caller=checkpoint.go:378 msg="writing series" size="385 kB"
level=info ts=2026-02-16T19:09:47.088945037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:09:47.09229167Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T19:09:47.099075716Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T19:09:47.511726646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:41996"
level=warn ts=2026-02-16T19:09:48.07189257Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T19:09:49.591865064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:51724"
level=info ts=2026-02-16T19:09:50.068507593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:09:50.07207253Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:09:50.072092874Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:09:50.372939764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:09:50.376855883Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T19:09:50.572168528Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=info ts=2026-02-16T19:09:52.54962448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="483 kB" avg_comp="96 kB" total_uncomp="4.9 MB" avg_uncomp="981 kB" full=4 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:09:52.553375706Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T19:09:53.061919752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T19:09:53.065452925Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=warn ts=2026-02-16T19:09:53.071619088Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:09:53.653819592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="384 B" total_uncomp="1.7 kB" avg_uncomp="347 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:09:53.657391382Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T19:09:54.321611209Z caller=checkpoint.go:378 msg="writing series" size="113 kB"
level=info ts=2026-02-16T19:09:56.009108958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:09:56.012722267Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T19:09:56.977443255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:09:56.980916931Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T19:09:58.071273279Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:09:58.071319858Z caller=checkpoint.go:378 msg="writing series" size="161 kB"
level=info ts=2026-02-16T19:10:00.101759113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T19:10:00.105587847Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T19:10:01.823122405Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T19:10:01.823184882Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=6
level=info ts=2026-02-16T19:10:02.340110146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:10:02.34666411Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T19:10:03.071191062Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:10:03.227607808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:10:03.231304943Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T19:10:03.810765714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:10:03.815863085Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T19:10:04.600359657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:10:04.605141446Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T19:10:05.150200498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.7 kB" avg_comp="861 B" total_uncomp="6.9 kB" avg_uncomp="3.4 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T19:10:05.153769061Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T19:10:05.179857353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="873 kB" avg_comp="873 kB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:10:05.184370057Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c67a47458%3A19c67d5cd86%3Ad6d8ff59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:10:05.184396415Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:10:05.184430843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="873 kB" avg_comp="873 kB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:10:05.188577729Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c67a47458%3A19c67d5cd86%3Ad6d8ff59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T19:10:05.19274972Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:10:05.236825952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:10:05.240252411Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T19:10:05.571604772Z caller=checkpoint.go:378 msg="writing series" size="443 kB"
level=info ts=2026-02-16T19:10:06.350703038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:10:06.354912214Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-02-16T19:10:06.581737127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:32770"
level=info ts=2026-02-16T19:10:07.589419555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:10:07.593313199Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T19:10:08.07183997Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:10:08.368923671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:10:08.372378504Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T19:10:09.322305119Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB"
level=debug ts=2026-02-16T19:10:09.322332781Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=2
ts=2026-02-16T19:10:11.434393989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-fmppm-a0176dcd 10.129.0.75:7946"
level=info ts=2026-02-16T19:10:12.433666762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T19:10:12.437202969Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T19:10:12.865594846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-02-16T19:10:12.869763994Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19c67a467a5%3A19c67d4c60a%3A148796e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-02-16T19:10:13.053546558Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.59:11211 count=1
level=info ts=2026-02-16T19:10:13.054577785Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T19:10:13.071816223Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:10:13.071839755Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:10:13.071848594Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T19:10:13.071854415Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:10:13.072436299Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T19:10:13.072451904Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=1
level=info ts=2026-02-16T19:10:14.485339821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="47 kB" avg_comp="12 kB" total_uncomp="306 kB" avg_uncomp="76 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:10:14.488799307Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T19:10:16.82132305Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-02-16T19:10:18.071716581Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:10:19.247428688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T19:10:19.252765479Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
ts=2026-02-16T19:10:19.595073651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:55456"
level=info ts=2026-02-16T19:10:19.940048903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="873 kB" avg_comp="873 kB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:10:19.944646848Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c67a47458%3A19c67d5cd86%3Ad6d8ff59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T19:10:20.535326489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="839 B" avg_comp="279 B" total_uncomp="732 B" avg_uncomp="244 B" full=3 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:10:20.539068681Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c67a4d66d%3A19c67a4f0bd%3A8064cec6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T19:10:20.571578497Z caller=checkpoint.go:378 msg="writing series" size="446 kB"
level=info ts=2026-02-16T19:10:20.674813934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:10:20.678327186Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T19:10:23.071596565Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:10:24.321670681Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T19:10:25.04757924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:10:25.051465613Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T19:10:25.611050845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:10:25.614663338Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T19:10:27.268169974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=3 total_comp="4.1 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="5.5 kB" full=3 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T19:10:27.2716642Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c67a4d697%3A19c67a5a17e%3Aad77e43f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T19:10:28.071911866Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-02-16T19:10:28.071907329Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:10:29.039417187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:10:29.042700054Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:10:29.042722523Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:10:29.04274404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:10:29.045270681Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T19:10:29.465771584Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T19:10:31.826261022Z caller=checkpoint.go:378 msg="writing series" size="8.1 MB"
level=debug ts=2026-02-16T19:10:31.826292996Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.5 MB" series=5
level=warn ts=2026-02-16T19:10:33.071908547Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:10:33.684729793Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T19:10:35.571884312Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=info ts=2026-02-16T19:10:36.403271348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:10:36.407083926Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:10:36.407104588Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:10:36.407126703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:10:36.409959313Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T19:10:38.071565675Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:10:38.454975071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="10 kB" avg_comp="3.5 kB" total_uncomp="48 kB" avg_uncomp="16 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:10:38.458395614Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c67a46774%3A19c67b6cc06%3A81bf4cc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T19:10:38.812743504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:10:38.816713366Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:10:38.816738549Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:10:38.816764111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:10:38.82006312Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T19:10:39.321936373Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
level=info ts=2026-02-16T19:10:41.054979053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.7 kB" avg_comp="3.7 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:10:41.058374684Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T19:10:41.43737061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49b608d9 10.129.0.76:7946"
level=info ts=2026-02-16T19:10:42.543779668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:10:42.547579141Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c67a4d66c%3A19c67a4d698%3A23726d0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T19:10:42.56659261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:10:42.570187366Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c67a4d66d%3A19c67a4f0bd%3A2115b6a1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T19:10:43.071091433Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T19:10:43.071097418Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:10:43.071125202Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:10:43.071134495Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T19:10:43.071255513Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=info ts=2026-02-16T19:10:43.248715271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=4 total_comp="4.0 kB" avg_comp="997 B" total_uncomp="9.9 kB" avg_uncomp="2.5 kB" full=4 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T19:10:43.252293562Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c67a4d697%3A19c67a4d698%3A549e6178?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T19:10:44.544484579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.5 MB" avg_uncomp="6.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:10:44.549788335Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c67a46c15%3A19c67a97c02%3A1f8829c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T19:10:46.093514252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:10:46.097315552Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c67a4d697%3A19c67a4f0bd%3Ac1a2c748?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T19:10:46.822682545Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T19:10:46.822710408Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-02-16T19:10:46.980673885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bccff7819963388a immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-obvk\"}"
level=error ts=2026-02-16T19:10:46.984560133Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bccff7819963388a/19c67b2cc95%3A19c67bb09f6%3Ac2e8d8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-obvk\"}"
level=info ts=2026-02-16T19:10:47.794579058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="873 kB" avg_comp="873 kB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:10:47.799903299Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c67a47458%3A19c67d5cd86%3Ad6d8ff59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-02-16T19:10:48.070961476Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:10:49.397432483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.7 kB" avg_comp="861 B" total_uncomp="6.9 kB" avg_uncomp="3.4 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T19:10:49.402507946Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c67a4d697%3A19c67a4de0f%3Ac5650fc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
ts=2026-02-16T19:10:49.598290745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:34180"
level=info ts=2026-02-16T19:10:49.879721723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:10:49.883672319Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c67a4d697%3A19c67a52097%3A38b017de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T19:10:50.571675242Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T19:10:51.162465423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="483 kB" avg_comp="96 kB" total_uncomp="4.9 MB" avg_uncomp="981 kB" full=4 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:10:51.173100094Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c67a46dba%3A19c67a9d9c3%3Acb5541d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T19:10:51.376710884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="384 B" total_uncomp="1.7 kB" avg_uncomp="347 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:10:51.380479339Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67aeac22%3A19c67aeac23%3Af930a892?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T19:10:53.071594485Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:10:53.399799098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:10:53.403485869Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c67a4d66d%3A19c67a4f0bd%3A95524f7e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T19:10:54.321709867Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T19:10:54.453372023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:10:54.457146729Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67b2a7a9%3A19c67b4a53e%3Ac72c0bc4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T19:10:55.881360441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="395 kB" avg_uncomp="395 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:10:55.885371245Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c67a46c02%3A19c67a75ae7%3A71cdf179?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T19:10:56.880269957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-02-16T19:10:56.884440221Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19c67a467a5%3A19c67d4c60a%3A148796e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-02-16T19:10:58.071186634Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:10:58.07128838Z caller=checkpoint.go:378 msg="writing series" size="196 kB"
level=info ts=2026-02-16T19:10:58.160455347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:10:58.166459331Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:10:58.166482951Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:10:58.166508705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:10:58.172734926Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c67a467bc%3A19c67a60a08%3Aec3315bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T19:10:58.367656063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=3 total_comp="18 kB" avg_comp="6.0 kB" total_uncomp="99 kB" avg_uncomp="33 kB" full=3 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:10:58.371599105Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c67a4d697%3A19c67a5e7ce%3Aca385703?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T19:10:58.399714934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="767 kB" avg_comp="767 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:10:58.404484021Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c67a46dd4%3A19c67c1e6c0%3Ad094c2a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T19:11:01.11620903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="47 kB" avg_comp="12 kB" total_uncomp="306 kB" avg_uncomp="76 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:11:01.122917362Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c67a47369%3A19c67b5a932%3A83569207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T19:11:01.669399536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:11:01.672660194Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c67a4de0e%3A19c67a5325a%3A83818340?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T19:11:01.822089616Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
level=warn ts=2026-02-16T19:11:03.07126173Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:11:03.810216246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0e88b81949e21b28 immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-sffa\"}"
level=error ts=2026-02-16T19:11:03.813889847Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e88b81949e21b28/19c67b2ab91%3A19c67bc0fae%3Ae6bd10ba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-sffa\"}"
level=debug ts=2026-02-16T19:11:05.571426622Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-02-16T19:11:06.104722997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="62 kB" avg_uncomp="62 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T19:11:06.108635526Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c67a4d697%3A19c67adc76e%3A793eb30d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
ts=2026-02-16T19:11:06.586209705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51140"
level=debug ts=2026-02-16T19:11:07.430440909Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T19:11:08.071941958Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:11:08.687621822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:11:08.691539205Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c67a4de0d%3A19c67a6f16d%3A9c3c105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T19:11:09.321903277Z caller=checkpoint.go:378 msg="writing series" size="228 kB"
level=info ts=2026-02-16T19:11:10.083119093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="605 B" avg_comp="605 B" total_uncomp="487 B" avg_uncomp="487 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:11:10.086862128Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a55746%3A19c67a5a566%3A31b0dcab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T19:11:11.429076103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.0 kB" avg_comp="520 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:11:11.433599093Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c67a4f0bc%3A19c67a4f0bd%3A31fa1c98?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T19:11:11.439811351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-g9mm5-d2693f97 10.129.0.72:7946"