level=info ts=2026-02-16T05:56:33.494972891Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-02-16T05:56:33.495006529Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-02-16T05:56:33.495336101Z caller=modules.go:1984 msg="initializing ingester query tags interceptors"
level=info ts=2026-02-16T05:56:33.496084193Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-02-16T05:56:33.496468735Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T05:56:33.496548235Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.62 inf=eth0
level=error ts=2026-02-16T05:56:33.501559695Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T05:56:33.501582512Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-02-16T05:56:33.50356368Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-02-16T05:56:33.503631781Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T05:56:33.503886348Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-02-16T05:56:33.50389844Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-02-16T05:56:33.503897547Z
level=debug ts=2026-02-16T05:56:33.503937809Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-02-16T05:56:33.503951701Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-02-16T05:56:33.503962888Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-02-16T05:56:33.503967221Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-02-16T05:56:33.503966664Z
level=debug ts=2026-02-16T05:56:33.503971221Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-02-16T05:56:33.504697676Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-6332ef80
ts=2026-02-16T05:56:33.504730414Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-02-16T05:56:33.504741488Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.62 advertisePort=7946
level=debug ts=2026-02-16T05:56:33.504759399Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.62 advertisePort=7946
level=info ts=2026-02-16T05:56:33.50487641Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=debug ts=2026-02-16T05:56:33.505752225Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T05:56:33.505823414Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.129.0.62 inf=eth0
ts=2026-02-16T05:56:33.508196626Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T05:56:33.508209224Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T05:56:33.508218148Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=3.343838ms
level=error ts=2026-02-16T05:56:33.508224964Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-02-16T05:56:33.508234471Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-02-16T05:56:33.510672833Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T05:56:33.51068583Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T05:56:33.519904133Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-02-16T05:56:33.519957959Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-02-16T05:56:33.519987381Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=debug ts=2026-02-16T05:56:33.519998103Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-02-16T05:56:33.520002723Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=debug ts=2026-02-16T05:56:33.520012707Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-02-16T05:56:33.520018352Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-02-16T05:56:33.520026728Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-02-16T05:56:33.520028408Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-02-16T05:56:33.520035203Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=debug ts=2026-02-16T05:56:33.520040126Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-02-16T05:56:33.520043148Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-02-16T05:56:33.520041489Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=info ts=2026-02-16T05:56:33.520235083Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2026-02-16T05:56:33.520239134Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-02-16T05:56:33.520248171Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-02-16T05:56:33.520266658Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=info ts=2026-02-16T05:56:33.520262415Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-02-16T05:56:33.520294796Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-02-16T05:56:33.520299556Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=info ts=2026-02-16T05:56:33.520302733Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-02-16T05:56:33.520393798Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-02-16T05:56:33.520398809Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-02-16T05:56:33.520402503Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=info ts=2026-02-16T05:56:33.520405427Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-02-16T05:56:33.520419843Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint"
level=info ts=2026-02-16T05:56:33.520449591Z caller=recovery.go:42 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-02-16T05:56:33.520495671Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=76.714µs errors=false
level=info ts=2026-02-16T05:56:33.520504469Z caller=ingester.go:587 component=ingester msg="recovering from WAL"
level=info ts=2026-02-16T05:56:33.520660937Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=242.18µs errors=false
level=info ts=2026-02-16T05:56:33.520668662Z caller=ingester.go:551 component=ingester msg="closing recoverer"
level=info ts=2026-02-16T05:56:33.520677795Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=259.145µs
level=info ts=2026-02-16T05:56:33.52069636Z caller=wal.go:158 msg=started component=wal
level=info ts=2026-02-16T05:56:33.520721411Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-02-16T05:56:33.520719024Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=8.918519252s
level=info ts=2026-02-16T05:56:33.520739585Z caller=lifecycler.go:714 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=debug ts=2026-02-16T05:56:33.520814687Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-02-16T05:56:33.520819997Z caller=lifecycler.go:556 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=info ts=2026-02-16T05:56:33.520821195Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 1m0s with no heartbeat"
level=info ts=2026-02-16T05:56:33.520841792Z caller=loki.go:599 msg="Loki started" startup_time=86.431645ms
level=debug ts=2026-02-16T05:56:33.520906108Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-02-16T05:56:34.697890649Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T05:56:34.697914278Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T05:56:38.116147031Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T05:56:38.11618863Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T05:56:38.521633106Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T05:56:42.952882596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.57:7946"
ts=2026-02-16T05:56:42.95445232Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.58:7946: dial tcp 10.128.0.58:7946: connect: connection refused"
ts=2026-02-16T05:56:42.955119732Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.59:7946: dial tcp 10.128.0.59:7946: connect: connection refused"
level=info ts=2026-02-16T05:56:42.955137084Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=9.4468984s
level=warn ts=2026-02-16T05:56:43.521397867Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T05:56:48.521794482Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T05:56:52.731341745Z caller=logging.go:137 msg="GET /ready (503) 90.246µs"
level=warn ts=2026-02-16T05:56:53.521193543Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T05:56:58.52178005Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T05:57:02.730540632Z caller=logging.go:137 msg="GET /ready (503) 70.847µs"
level=warn ts=2026-02-16T05:57:03.521482245Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T05:57:03.521482176Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-02-16T05:57:03.521511322Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-02-16T05:57:03.521516528Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-02-16T05:57:05.702960944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=warn ts=2026-02-16T05:57:08.521851672Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T05:57:13.521260957Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T05:57:15.537743364Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T05:57:18.521384802Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T05:57:23.521501469Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T05:57:28.521282221Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T05:57:32.508407534Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T05:57:33.502947268Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T05:57:33.503987508Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T05:57:33.521780204Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T05:57:33.521800035Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T05:57:33.52180898Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T05:57:33.521809346Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T05:57:35.704569728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T05:57:36.11290361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:37880"
ts=2026-02-16T05:57:38.010748294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:56702"
level=warn ts=2026-02-16T05:57:38.521595998Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T05:57:43.521737877Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T05:57:48.521574015Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T05:57:50.288460308Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T05:57:51.510263942Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T05:57:53.521646741Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T05:57:58.52181692Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T05:58:03.520873004Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T05:58:03.520896482Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T05:58:03.520921746Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T05:58:03.520930316Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-02-16T05:58:05.705252782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=warn ts=2026-02-16T05:58:08.521869791Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T05:58:11.841393579Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T05:58:13.521628619Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T05:58:15.112880147Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T05:58:18.521194435Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T05:58:23.52087557Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T05:58:28.521650463Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T05:58:31.129080518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:40766"
level=debug ts=2026-02-16T05:58:32.230579793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T05:58:33.502975497Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T05:58:33.504022487Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T05:58:33.521763015Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T05:58:33.521780808Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T05:58:33.521788935Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T05:58:33.52179512Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T05:58:35.706470963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T05:58:36.115340803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:52416"
ts=2026-02-16T05:58:38.01448718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:37820"
level=warn ts=2026-02-16T05:58:38.521815314Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T05:58:43.521541331Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T05:58:44.956321646Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T05:58:48.521426914Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T05:58:53.521002918Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T05:58:58.521104782Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T05:59:03.52089433Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T05:59:03.520920998Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T05:59:03.520930465Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T05:59:03.520932165Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T05:59:05.707887177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T05:59:06.252916152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T05:59:06.254628497Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T05:59:08.521243358Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T05:59:13.521003103Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T05:59:17.260145455Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T05:59:17.260218369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T05:59:17.271918301Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T05:59:18.520924265Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T05:59:23.521395528Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T05:59:28.521109753Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T05:59:30.21629127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T05:59:30.223888125Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T05:59:31.132266616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:57304"
level=debug ts=2026-02-16T05:59:33.503645906Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T05:59:33.503679799Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T05:59:33.52171953Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T05:59:33.521742047Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T05:59:33.521752639Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T05:59:33.521752502Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T05:59:35.709217391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=warn ts=2026-02-16T05:59:38.521421601Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T05:59:43.521720744Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T05:59:45.743979382Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T05:59:47.1478066Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T05:59:48.521730354Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T05:59:53.521866935Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T05:59:58.520928253Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T06:00:01.134029029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:44620"
level=info ts=2026-02-16T06:00:01.226547317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:00:01.237000343Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:00:03.52133996Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:00:03.521382451Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:00:03.52139115Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:00:03.521404535Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T06:00:05.710452841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:00:08.019874404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:40244"
level=warn ts=2026-02-16T06:00:08.521212107Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T06:00:13.521018552Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:00:14.519400174Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T06:00:18.521347763Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T06:00:23.521049119Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T06:00:28.52179338Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:00:33.503537934Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:00:33.50460053Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T06:00:33.504804402Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-02-16T06:00:33.504850835Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1968023 n=1
level=debug ts=2026-02-16T06:00:33.50489476Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1968023 n=1
level=info ts=2026-02-16T06:00:33.520893177Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T06:00:33.52090622Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:00:33.520920266Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:00:33.520928417Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-02-16T06:00:35.712204602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:00:36.122726489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:47318"
ts=2026-02-16T06:00:38.022013825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:47242"
level=warn ts=2026-02-16T06:00:38.521143758Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T06:00:43.521252116Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:00:44.1390685Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T06:00:48.521103685Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:00:51.526351191Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T06:00:52.02656556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:00:52.032706778Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T06:00:53.521603623Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T06:00:58.521530567Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T06:01:03.521781901Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:01:03.521790537Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:01:03.521811328Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:01:03.521819433Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-02-16T06:01:05.713909481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=warn ts=2026-02-16T06:01:08.521336692Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:01:08.539516195Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T06:01:13.521200675Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T06:01:18.521126149Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T06:01:23.521457907Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T06:01:28.520879604Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:01:33.502763465Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:01:33.503828244Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:01:33.521294916Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-02-16T06:01:33.521299163Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:01:33.521384861Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:01:33.521401976Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:01:33.521409818Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T06:01:33.521447702Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=debug ts=2026-02-16T06:01:33.521642549Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
ts=2026-02-16T06:01:35.716254579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=info ts=2026-02-16T06:01:35.971721134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:01:35.978867146Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T06:01:36.12600555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:38522"
level=debug ts=2026-02-16T06:01:37.948303652Z caller=checkpoint.go:378 msg="writing series" size="95 kB"
level=warn ts=2026-02-16T06:01:38.521228896Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:01:41.710509174Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T06:01:42.37445842Z caller=checkpoint.go:378 msg="writing series" size="298 kB"
level=warn ts=2026-02-16T06:01:43.521435251Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:01:44.83288071Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T06:01:46.801219458Z caller=checkpoint.go:378 msg="writing series" size="149 kB"
level=warn ts=2026-02-16T06:01:48.520876641Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:01:51.227018665Z caller=checkpoint.go:378 msg="writing series" size="127 kB"
level=warn ts=2026-02-16T06:01:53.521726362Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:01:55.652848436Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=warn ts=2026-02-16T06:01:58.521716991Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:02:00.080117037Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
ts=2026-02-16T06:02:01.141185553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:38166"
level=info ts=2026-02-16T06:02:03.521595965Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:02:03.521625807Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:02:03.521639334Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:02:03.521650504Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:02:04.505330543Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
ts=2026-02-16T06:02:05.718304232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:02:06.127610134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:57112"
level=warn ts=2026-02-16T06:02:08.520988528Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:02:08.931526885Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=info ts=2026-02-16T06:02:09.449851461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="16 kB" avg_uncomp="16 kB" idle=1 labels="{stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:02:09.449922828Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-02-16T06:02:09.45451902Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:02:13.358380743Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-02-16T06:02:13.521649374Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:02:16.808883312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:02:16.815560113Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:02:17.784113818Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=warn ts=2026-02-16T06:02:18.521013969Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:02:22.210979351Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-02-16T06:02:23.521387194Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:02:26.636763309Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-02-16T06:02:28.521078516Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:02:29.048088621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:02:29.054534295Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:02:31.063226877Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=debug ts=2026-02-16T06:02:31.063249575Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=14
level=debug ts=2026-02-16T06:02:33.502974931Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:02:33.504039315Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:02:33.521123629Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:02:33.521149744Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:02:33.521158622Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:02:33.521202489Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:02:35.489711774Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
ts=2026-02-16T06:02:35.720235741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=warn ts=2026-02-16T06:02:38.521197621Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:02:39.915563907Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=info ts=2026-02-16T06:02:40.241454438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="158 B" avg_comp="158 B" total_uncomp="86 kB" avg_uncomp="86 kB" idle=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T06:02:40.241526238Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-02-16T06:02:40.246030794Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:02:40.541891304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="126 B" avg_comp="126 B" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" idle=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:02:40.545119773Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T06:02:40.842043408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="3.1 kB" avg_comp="3.1 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:02:40.845591345Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:02:41.142499839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="28 kB" avg_uncomp="28 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T06:02:41.145641168Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c65062957%3A19c65062958%3Ae78c1a55?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T06:02:41.441958287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" idle=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:02:41.445094234Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c65062bfa%3A19c65062bfb%3A13a6e500?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T06:02:41.742419224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="274 B" avg_comp="274 B" total_uncomp="274 B" avg_uncomp="274 B" idle=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:02:41.745411663Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:02:42.04169422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:02:42.044853059Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:02:42.341772357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:02:42.344811719Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:02:42.641527157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="5.0 kB" avg_comp="5.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:02:42.644611895Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:02:42.942428945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="2.4 kB" avg_comp="2.4 kB" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:02:42.945782186Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:02:43.241677345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" idle=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:02:43.244936628Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T06:02:43.520969193Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:02:43.542134785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="54 kB" avg_uncomp="54 kB" idle=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:02:43.545257538Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T06:02:44.34180876Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=warn ts=2026-02-16T06:02:48.521043707Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:02:48.767586756Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
level=debug ts=2026-02-16T06:02:53.194797577Z caller=checkpoint.go:378 msg="writing series" size="1.6 kB"
level=warn ts=2026-02-16T06:02:53.521118693Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:02:53.888429534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:02:53.893267534Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:02:54.773235107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:02:54.776705726Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:02:54.863055584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:02:54.86637887Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:02:55.219412861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:02:55.222498044Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T06:02:55.236554411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:02:55.239233934Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:02:55.866883772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:02:55.869947072Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T06:02:56.844483567Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T06:02:57.336213265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:02:57.33943106Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T06:02:57.620135856Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=info ts=2026-02-16T06:02:58.09543074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:02:58.098588407Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T06:02:58.521409279Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:02:59.585790323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:02:59.589244222Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:03:00.162826932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T06:03:00.166249383Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c65062957%3A19c65062958%3Ae78c1a55?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T06:03:01.362966488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:03:01.366379028Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c65062bfa%3A19c65062bfb%3A13a6e500?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T06:03:02.014586529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:03:02.017950559Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T06:03:02.047040711Z caller=checkpoint.go:378 msg="writing series" size="77 kB"
level=info ts=2026-02-16T06:03:03.106401731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:03:03.109887238Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T06:03:03.520996969Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:03:03.521025659Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:03:03.521049405Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:03:03.5210571Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:03:04.97415709Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:03:04.974218875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="153 B" avg_comp="153 B" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:03:04.977803833Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:03:05.27451277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="10 kB" avg_uncomp="10 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T06:03:05.278307403Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c65066aed%3A19c650697d1%3Af3ec1025?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T06:03:05.575151833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="7.4 kB" avg_comp="7.4 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" idle=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:03:05.578421664Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
ts=2026-02-16T06:03:05.722157793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:03:06.473466777Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T06:03:06.473491411Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=8
level=warn ts=2026-02-16T06:03:08.521390044Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:03:09.669556192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:03:09.675897755Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:03:10.89923333Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-02-16T06:03:13.521021608Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:03:15.325153817Z caller=checkpoint.go:378 msg="writing series" size="166 kB"
level=info ts=2026-02-16T06:03:15.811485277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:03:15.815563585Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:03:16.122488669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:03:16.126281088Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:03:18.254632354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:03:18.257983409Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T06:03:18.521143723Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:03:19.751967751Z caller=checkpoint.go:378 msg="writing series" size="437 kB"
level=info ts=2026-02-16T06:03:20.264003062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:03:20.267199149Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T06:03:20.754543765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T06:03:20.760089941Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c65062957%3A19c65062958%3Ae78c1a55?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T06:03:20.85127323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:03:20.854262381Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:03:20.875332827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T06:03:20.878032305Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c65066aed%3A19c650697d1%3Af3ec1025?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T06:03:21.752258116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:03:21.755466015Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T06:03:21.892740367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:03:21.895862075Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T06:03:23.521890879Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:03:24.178115116Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=info ts=2026-02-16T06:03:24.732669672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:03:24.736550069Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T06:03:26.688729102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:03:26.692034469Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c65062bfa%3A19c65062bfb%3A13a6e500?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T06:03:28.521152086Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:03:28.604463936Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T06:03:30.869157175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:03:30.872522544Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T06:03:33.030278721Z caller=checkpoint.go:378 msg="writing series" size="7.2 kB"
level=debug ts=2026-02-16T06:03:33.502982002Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:03:33.504043611Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T06:03:33.521733316Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:03:33.521737956Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:03:33.521756655Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:03:33.52176684Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-02-16T06:03:35.72334403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=info ts=2026-02-16T06:03:35.829156087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:03:35.832646106Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:03:35.881699783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:03:35.885180079Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:03:36.081858842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" idle=1 labels="{stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T06:03:36.081879624Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-02-16T06:03:36.085608584Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T06:03:36.133007306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:42276"
level=debug ts=2026-02-16T06:03:37.456865731Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=warn ts=2026-02-16T06:03:38.521405062Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:03:41.633784769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:03:41.639109109Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T06:03:41.882384732Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-02-16T06:03:43.520997664Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:03:44.061408776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:03:44.064819738Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:03:44.277408287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:03:44.281444293Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T06:03:46.308952108Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-02-16T06:03:48.52172326Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:03:50.228306013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T06:03:50.231819562Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c65066aed%3A19c650697d1%3Af3ec1025?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-02-16T06:03:50.738168103Z caller=checkpoint.go:378 msg="writing series" size="4.0 MB"
level=debug ts=2026-02-16T06:03:50.738189856Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.0 MB" series=10
level=warn ts=2026-02-16T06:03:53.521160491Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:03:53.597462733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:03:53.604272701Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:03:55.162086928Z caller=checkpoint.go:378 msg="writing series" size="299 kB"
level=info ts=2026-02-16T06:03:56.043414153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:03:56.046667668Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:03:58.242741131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:03:58.245875753Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T06:03:58.521403917Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:03:59.587586541Z caller=checkpoint.go:378 msg="writing series" size="512 B"
ts=2026-02-16T06:04:01.149000232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:35048"
level=info ts=2026-02-16T06:04:03.521523494Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:04:03.521547449Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:04:03.521555125Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:04:03.52158299Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:04:04.013796532Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T06:04:05.352069614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:04:05.35530316Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-02-16T06:04:05.724681226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=info ts=2026-02-16T06:04:07.38611262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:04:07.389095561Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:04:07.623313997Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T06:04:07.624836051Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T06:04:08.44010892Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-02-16T06:04:08.521187631Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:04:10.393004368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:04:10.396250362Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c65062bfa%3A19c65062bfb%3A13a6e500?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T06:04:11.024987989Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:04:11.025053565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="146 B" avg_comp="146 B" total_uncomp="22 kB" avg_uncomp="22 kB" idle=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:04:11.028135408Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:04:11.080206969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="323 B" avg_comp="323 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:04:11.08311862Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:04:12.102939774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="622 B" avg_comp="622 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:04:12.107083372Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T06:04:12.866478054Z caller=checkpoint.go:378 msg="writing series" size="260 kB"
level=warn ts=2026-02-16T06:04:13.521717992Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:04:14.284877736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:04:14.288158745Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T06:04:17.078050231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:04:17.081396135Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T06:04:17.293026424Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
level=warn ts=2026-02-16T06:04:18.521400472Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:04:18.883469838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:04:18.886806982Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:04:19.558396077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T06:04:19.561581691Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c65062957%3A19c65062958%3Ae78c1a55?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T06:04:19.628834319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:04:19.631593358Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:04:21.190904941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:04:21.194115286Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T06:04:21.718858523Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-02-16T06:04:23.521032283Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:04:26.145745414Z caller=checkpoint.go:378 msg="writing series" size="239 kB"
level=info ts=2026-02-16T06:04:26.397639068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:04:26.401131131Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:04:27.291682473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:04:27.295068771Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T06:04:28.521379766Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:04:30.571405197Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=debug ts=2026-02-16T06:04:33.503385849Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:04:33.504384391Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:04:33.521197135Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:04:33.521215795Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:04:33.521224582Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:04:33.521227214Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:04:34.998053649Z caller=checkpoint.go:378 msg="writing series" size="240 kB"
level=debug ts=2026-02-16T06:04:34.998078861Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=10
level=info ts=2026-02-16T06:04:35.221611514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:04:35.225340451Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
ts=2026-02-16T06:04:35.726457672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:04:38.036285239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:42114"
level=info ts=2026-02-16T06:04:38.168975853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:04:38.175280861Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T06:04:38.521682404Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:04:38.793345805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T06:04:38.796603491Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c65066aed%3A19c650697d1%3Af3ec1025?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-02-16T06:04:39.423843522Z caller=checkpoint.go:378 msg="writing series" size="98 kB"
level=info ts=2026-02-16T06:04:39.508302616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:04:39.51150621Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T06:04:40.12146289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:04:40.124989535Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T06:04:43.00734329Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T06:04:43.521471367Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:04:43.850500085Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=debug ts=2026-02-16T06:04:48.276230449Z caller=checkpoint.go:378 msg="writing series" size="157 kB"
level=warn ts=2026-02-16T06:04:48.521267757Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:04:48.898972658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:04:48.904082514Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:04:51.920856867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:04:51.924221662Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T06:04:52.702164621Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T06:04:52.814627046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:04:52.818237262Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c65062bfa%3A19c65062bfb%3A13a6e500?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T06:04:53.520902288Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:04:56.831491258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:04:56.835117231Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:04:57.129238024Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-02-16T06:04:58.521427504Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:04:59.192477584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:04:59.195932293Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-02-16T06:05:01.153530083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:38536"
level=info ts=2026-02-16T06:05:01.5399848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:05:01.543431267Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T06:05:01.554732814Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=info ts=2026-02-16T06:05:03.52128494Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T06:05:03.521292653Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:05:03.521308391Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:05:03.521316238Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:05:03.63638486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:05:05.728830717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:05:05.981317702Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-02-16T06:05:06.833552649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:05:06.836778652Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:05:08.18848349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:05:08.191922993Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T06:05:08.521469515Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:05:08.861527738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:05:08.864663803Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T06:05:10.406955386Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=info ts=2026-02-16T06:05:12.744024957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:05:12.747995038Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T06:05:13.193085265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:05:13.196718645Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T06:05:13.521242408Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:05:14.722404099Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:05:14.722474489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="145 kB" avg_comp="145 kB" total_uncomp="358 kB" avg_uncomp="358 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:05:14.726007941Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:05:14.833302992Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=info ts=2026-02-16T06:05:15.983570181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T06:05:15.986951367Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c65062957%3A19c65062958%3Ae78c1a55?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T06:05:18.521216919Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:05:19.259791903Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=info ts=2026-02-16T06:05:23.428847439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:05:23.433394445Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T06:05:23.521162937Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:05:23.686565848Z caller=checkpoint.go:378 msg="writing series" size="239 kB"
level=info ts=2026-02-16T06:05:23.758405077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="928 B" avg_comp="928 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:05:23.762044682Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:05:23.947464835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:05:23.950796677Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:05:24.350867098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:05:24.357107339Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:05:24.357128049Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:05:24.357152781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:05:24.362716766Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:05:27.055257849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:05:27.058571948Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:05:28.112476366Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-02-16T06:05:28.521011465Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:05:30.961514195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:05:30.965261529Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T06:05:32.538461632Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=info ts=2026-02-16T06:05:33.210631247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T06:05:33.214149727Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c65066aed%3A19c650697d1%3Af3ec1025?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T06:05:33.420752193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:05:33.424226785Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:05:33.503664048Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T06:05:33.503665946Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:05:33.521221337Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:05:33.521240154Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:05:33.521249022Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:05:33.521277093Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:05:35.221276752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:05:35.224869107Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
ts=2026-02-16T06:05:35.730779144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:05:36.140475653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:46720"
level=info ts=2026-02-16T06:05:36.185028604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:05:36.19201616Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:05:36.965233278Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-02-16T06:05:38.52160198Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:05:38.739451277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:05:38.743054768Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:05:40.606523453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:05:40.610683827Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c65062bfa%3A19c65062bfb%3A13a6e500?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T06:05:41.390844271Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T06:05:41.508405456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:05:41.51217324Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T06:05:43.521492971Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:05:43.890175183Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:05:43.890237599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="345 B" avg_comp="345 B" total_uncomp="345 B" avg_uncomp="345 B" idle=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:05:43.893937694Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T06:05:45.817278095Z caller=checkpoint.go:378 msg="writing series" size="869 B"
level=info ts=2026-02-16T06:05:46.309538563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:05:46.313192383Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T06:05:48.521143772Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:05:50.243930216Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=info ts=2026-02-16T06:05:50.914170502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:05:50.917312424Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:05:53.278478076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:05:53.281809524Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T06:05:53.521309369Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:05:54.669960216Z caller=checkpoint.go:378 msg="writing series" size="391 kB"
level=debug ts=2026-02-16T06:05:54.669982268Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=18
level=debug ts=2026-02-16T06:05:56.613727866Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T06:05:58.521416044Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:05:59.096101546Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=info ts=2026-02-16T06:06:02.204307399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:06:02.208002498Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:06:03.482135578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="444 B" avg_comp="444 B" total_uncomp="345 B" avg_uncomp="345 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:06:03.485710249Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T06:06:03.520931677Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:06:03.520954407Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:06:03.520973251Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:06:03.520981274Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:06:03.522032963Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.8 kB" series=1
level=info ts=2026-02-16T06:06:03.527441174Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-02-16T06:06:03.529916236Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.008334275s
level=info ts=2026-02-16T06:06:05.007193232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:06:05.010905987Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:06:05.496305724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:06:05.500022961Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
ts=2026-02-16T06:06:05.732912291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=info ts=2026-02-16T06:06:05.761903014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:06:05.765506537Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-02-16T06:06:06.141600531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:46316"
level=info ts=2026-02-16T06:06:06.722967651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:06:06.72716575Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:06:08.265855169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:06:08.269503324Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T06:06:08.52110215Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:06:08.592202571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T06:06:08.595719237Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c65062957%3A19c65062958%3Ae78c1a55?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T06:06:10.387598392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:06:10.391137527Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-02-16T06:06:13.521324806Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:06:15.114467711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:06:15.120913006Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:06:16.669988516Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:06:17.260583393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:06:17.265314043Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T06:06:18.521754243Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:06:20.268592396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:06:20.272287627Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T06:06:23.520974838Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:06:23.666346504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:06:23.669963051Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:06:24.610335321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:06:24.615342815Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:06:26.978196556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:06:26.981904528Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:06:26.994955634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="462 B" avg_comp="462 B" total_uncomp="345 B" avg_uncomp="345 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:06:26.997920676Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T06:06:28.521722798Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:06:28.845484014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:06:28.849181382Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:06:30.574825937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T06:06:30.578672539Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c65066aed%3A19c650697d1%3Af3ec1025?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-02-16T06:06:33.503641398Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:06:33.50367523Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:06:33.520907906Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:06:33.520929489Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:06:33.520936957Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T06:06:33.520950453Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-02-16T06:06:33.520977464Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:06:33.521089391Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=debug ts=2026-02-16T06:06:33.521329877Z caller=checkpoint.go:378 msg="writing series" size="127 kB"
level=info ts=2026-02-16T06:06:35.470937947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:06:35.475016954Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c65062bfa%3A19c65062bfb%3A13a6e500?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
ts=2026-02-16T06:06:35.734973223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=info ts=2026-02-16T06:06:36.132326716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:06:36.135758915Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T06:06:37.94779968Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=warn ts=2026-02-16T06:06:38.521918242Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:06:39.222221664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:06:39.225632957Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:06:42.374463136Z caller=checkpoint.go:378 msg="writing series" size="301 kB"
level=warn ts=2026-02-16T06:06:43.521634955Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:06:46.800531707Z caller=checkpoint.go:378 msg="writing series" size="316 kB"
level=warn ts=2026-02-16T06:06:48.521708692Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:06:48.864630653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:06:48.869964588Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T06:06:50.737404394Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T06:06:51.226707673Z caller=checkpoint.go:378 msg="writing series" size="214 kB"
level=debug ts=2026-02-16T06:06:51.885242021Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T06:06:52.042735872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:06:52.046428134Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T06:06:53.521748077Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:06:54.112472492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.4 kB" avg_comp="2.4 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:06:54.115860661Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:06:55.652834952Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-02-16T06:06:58.521387752Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:06:59.279483741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T06:06:59.286160789Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c65062957%3A19c65062958%3Ae78c1a55?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T06:06:59.649168709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:06:59.652901739Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T06:07:00.079101706Z caller=checkpoint.go:378 msg="writing series" size="123 kB"
level=debug ts=2026-02-16T06:07:00.079130655Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-02-16T06:07:01.29776594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:07:01.30120548Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:07:02.959399997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:07:02.963277295Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:07:03.52155235Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:07:03.521582348Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:07:03.52159223Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:07:03.521595408Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:07:03.908918769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:07:03.916108597Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:07:04.481118988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:07:04.484921774Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T06:07:04.505144898Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-02-16T06:07:04.65258291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:07:04.656004633Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T06:07:05.736853091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=warn ts=2026-02-16T06:07:08.521015361Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:07:08.931329663Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=info ts=2026-02-16T06:07:12.819811292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:07:12.82358507Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:07:13.066353034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T06:07:13.069990984Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c65066aed%3A19c650697d1%3Af3ec1025?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-02-16T06:07:13.357880827Z caller=checkpoint.go:378 msg="writing series" size="203 kB"
level=warn ts=2026-02-16T06:07:13.521033285Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:07:14.777955587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:07:14.782007017Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:07:15.072836688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:07:15.076232931Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T06:07:16.974325975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:07:16.978166411Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T06:07:17.784419325Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T06:07:17.911683773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:07:17.91520841Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T06:07:18.521033723Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:07:18.636472373Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:07:20.575749087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="462 B" avg_comp="462 B" total_uncomp="345 B" avg_uncomp="345 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:07:20.580247393Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T06:07:22.210495948Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T06:07:22.352927733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:07:22.356449544Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T06:07:23.52107879Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:07:26.636506768Z caller=checkpoint.go:378 msg="writing series" size="4.4 kB"
level=warn ts=2026-02-16T06:07:28.521227881Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:07:31.062526781Z caller=checkpoint.go:378 msg="writing series" size="30 kB"
level=debug ts=2026-02-16T06:07:31.458709364Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T06:07:32.72227656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:07:32.727061185Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:07:33.503158672Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:07:33.504203898Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:07:33.52162174Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:07:33.521644363Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:07:33.521652936Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:07:33.52166163Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:07:34.573409716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:07:34.577353603Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:07:34.773921604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:07:34.7774187Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c65062bfa%3A19c65062bfb%3A13a6e500?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T06:07:34.895776051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:07:34.899473142Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T06:07:35.489265297Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
ts=2026-02-16T06:07:35.738802071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=info ts=2026-02-16T06:07:35.936290283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:07:35.940033705Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
ts=2026-02-16T06:07:36.146621121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:36604"
level=info ts=2026-02-16T06:07:37.251178126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:07:37.255002765Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T06:07:38.521128083Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:07:39.91582457Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=warn ts=2026-02-16T06:07:43.521302765Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:07:44.341317897Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-02-16T06:07:45.744583331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:07:45.748915013Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:07:45.906401332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:07:45.910430495Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:07:47.837277352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:07:47.840726107Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T06:07:48.024276342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:07:48.027762809Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T06:07:48.521691123Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:07:48.767588111Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
level=info ts=2026-02-16T06:07:50.619965075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T06:07:50.625980866Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c65062957%3A19c65062958%3Ae78c1a55?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T06:07:53.193770681Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=debug ts=2026-02-16T06:07:53.445674543Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T06:07:53.521600808Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:07:56.853960605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:07:56.85912004Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T06:07:57.620526088Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=info ts=2026-02-16T06:07:57.968096616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:07:57.971559593Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T06:07:58.520969639Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:07:58.951622633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:07:58.954944759Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
ts=2026-02-16T06:08:01.162961139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:47622"
level=debug ts=2026-02-16T06:08:02.046018422Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
level=info ts=2026-02-16T06:08:02.829562822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:08:02.833634627Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:08:03.521030949Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:08:03.521062274Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:08:03.521073897Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:08:03.521093888Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:08:03.761271692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:08:03.767804743Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T06:08:05.740892805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:08:06.473160054Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-02-16T06:08:06.473182324Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.8 MB" series=15
level=info ts=2026-02-16T06:08:07.225628212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T06:08:07.229188469Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c65066aed%3A19c650697d1%3Af3ec1025?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T06:08:08.254669065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:08:08.258170235Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T06:08:08.521698289Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:08:10.898401293Z caller=checkpoint.go:378 msg="writing series" size="4.0 kB"
level=info ts=2026-02-16T06:08:12.072510968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:08:12.077641521Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T06:08:13.521762308Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:08:14.435023356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:08:14.438812328Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:08:14.606412307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:08:14.609907302Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:08:15.252005077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="462 B" avg_comp="462 B" total_uncomp="345 B" avg_uncomp="345 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:08:15.255781321Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T06:08:15.325241554Z caller=checkpoint.go:378 msg="writing series" size="232 kB"
level=debug ts=2026-02-16T06:08:17.363492958Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:08:17.365291813Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T06:08:17.663199103Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T06:08:18.521316943Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:08:19.751774264Z caller=checkpoint.go:378 msg="writing series" size="563 kB"
level=info ts=2026-02-16T06:08:20.486326715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:08:20.490136163Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:08:22.092788791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:08:22.0980704Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T06:08:23.520919541Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:08:23.55610904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:08:23.560039046Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:08:24.177116303Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=info ts=2026-02-16T06:08:27.302574428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:08:27.306804006Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c65062bfa%3A19c65062bfb%3A13a6e500?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T06:08:28.521870915Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:08:28.604309117Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T06:08:30.004471586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:08:30.009415106Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T06:08:33.030026506Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=debug ts=2026-02-16T06:08:33.504180583Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:08:33.504190602Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:08:33.520807904Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:08:33.520831207Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:08:33.520839624Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:08:33.52087132Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:08:34.969727375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:08:34.973399187Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T06:08:34.973419849Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:08:34.973442172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:08:34.976482056Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
ts=2026-02-16T06:08:35.742517924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:08:37.460271732Z caller=checkpoint.go:378 msg="writing series" size="7.4 MB"
level=debug ts=2026-02-16T06:08:37.460293813Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.3 MB" series=7
level=warn ts=2026-02-16T06:08:38.521395025Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:08:38.800634957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:08:38.805174948Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T06:08:41.882177156Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-02-16T06:08:42.359071393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="571 B" avg_comp="571 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:08:42.363113224Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:08:42.618973562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:08:42.622408937Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T06:08:43.521858697Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:08:45.850744634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:08:45.854102197Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:08:46.308504191Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T06:08:46.474718542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:08:46.478074281Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:08:48.03963738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:08:48.043308314Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T06:08:48.52139049Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:08:49.890333309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T06:08:49.894409548Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c65062957%3A19c65062958%3Ae78c1a55?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T06:08:50.73624561Z caller=checkpoint.go:378 msg="writing series" size="4.6 MB"
level=debug ts=2026-02-16T06:08:50.736270785Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.6 MB" series=3
level=info ts=2026-02-16T06:08:51.632089403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:08:51.635689524Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:08:51.992580797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:08:51.999055513Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:08:52.257745792Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T06:08:53.52110216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:08:55.088018461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:08:55.092318959Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T06:08:55.161797819Z caller=checkpoint.go:378 msg="writing series" size="243 kB"
level=info ts=2026-02-16T06:08:56.008563139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="100 kB" avg_uncomp="50 kB" idle=1 full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:08:56.011808471Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T06:08:58.521169606Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:08:59.075655287Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:08:59.587598602Z caller=checkpoint.go:378 msg="writing series" size="512 B"
level=info ts=2026-02-16T06:08:59.812403296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:08:59.815442151Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T06:09:01.167735915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:52886"
level=info ts=2026-02-16T06:09:01.785889245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T06:09:01.789757051Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c65066aed%3A19c650697d1%3Af3ec1025?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T06:09:02.915270422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:09:02.91867266Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T06:09:02.918691901Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:09:03.219578079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:09:03.223044327Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T06:09:03.521578823Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:09:03.521584951Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:09:03.521605613Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:09:03.521614338Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:09:04.013311899Z caller=checkpoint.go:378 msg="writing series" size="6.4 kB"
ts=2026-02-16T06:09:05.744324307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=info ts=2026-02-16T06:09:05.92565135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="462 B" avg_comp="462 B" total_uncomp="345 B" avg_uncomp="345 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:09:05.929516695Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
ts=2026-02-16T06:09:06.151703789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:42040"
level=debug ts=2026-02-16T06:09:08.440345084Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-02-16T06:09:08.521794771Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:09:09.660869497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:09:09.664946374Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:09:09.664970207Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:09:09.664997917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:09:09.667869766Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:09:11.751755055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:09:11.755093746Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T06:09:11.755119902Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:09:11.755150402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:09:11.757917754Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T06:09:12.865855043Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=warn ts=2026-02-16T06:09:13.52179635Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:09:17.126549493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:09:17.13080169Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c65062bfa%3A19c65062bfb%3A13a6e500?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:09:17.130823044Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:09:17.130844785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:09:17.133430753Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c65062bfa%3A19c65062bfb%3A13a6e500?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T06:09:17.292836929Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=info ts=2026-02-16T06:09:17.918131647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:09:17.921793147Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T06:09:18.521735955Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:09:18.883164127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:09:18.884605255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:09:18.886486786Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:09:18.886487445Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:09:18.886502484Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=error ts=2026-02-16T06:09:18.886504633Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:09:18.886521245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:09:18.889153097Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T06:09:18.987508418Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:09:19.23335285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="323 B" avg_comp="323 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:09:19.236654964Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:09:19.236673667Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:09:19.487208533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:09:19.490488412Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:09:19.787383394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:09:19.7906968Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:09:21.718572145Z caller=checkpoint.go:378 msg="writing series" size="145 kB"
level=info ts=2026-02-16T06:09:22.598533237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="622 B" avg_comp="622 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:09:22.602380939Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T06:09:23.521559215Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:09:24.091872416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:09:24.095465886Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:09:26.144786059Z caller=checkpoint.go:378 msg="writing series" size="82 kB"
level=warn ts=2026-02-16T06:09:28.521453985Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:09:28.603752534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:09:28.608381289Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:09:29.670492846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:09:29.674299204Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:09:30.115992775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:09:30.119467782Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T06:09:30.570802752Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T06:09:32.537488548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:09:32.541238133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T06:09:32.541317525Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:09:32.543890128Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c65062957%3A19c65062958%3Ae78c1a55?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T06:09:32.543901747Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:09:32.54392118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="280 B" avg_comp="280 B" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" idle=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:09:32.546460271Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T06:09:33.503239208Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:09:33.504326207Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:09:33.521746779Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:09:33.521768738Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T06:09:33.521758214Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:09:33.521777105Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T06:09:34.194721505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:09:34.201104816Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:09:34.99748652Z caller=checkpoint.go:378 msg="writing series" size="205 kB"
level=info ts=2026-02-16T06:09:35.042495043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:09:35.045994634Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c65062bfa%3A19c65062bfb%3A13a6e500?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
ts=2026-02-16T06:09:35.746585332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=info ts=2026-02-16T06:09:37.160314675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:09:37.163645102Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:09:37.918619354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:09:37.922294171Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:09:37.922320356Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:09:37.922348316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:09:37.925512694Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-02-16T06:09:38.521065918Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:09:39.423576281Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=debug ts=2026-02-16T06:09:39.42361171Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-02-16T06:09:40.459260031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:09:40.462929489Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:09:40.462950946Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:09:40.462973148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:09:40.465819566Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:09:40.670533232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:09:40.674067371Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:09:40.674089731Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:09:40.693092266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:09:40.696278752Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:09:40.763490618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:09:40.76689828Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T06:09:43.521496472Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:09:43.849971053Z caller=checkpoint.go:378 msg="writing series" size="346 kB"
level=debug ts=2026-02-16T06:09:46.260762724Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:09:47.084954171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:09:47.090135548Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:09:48.003137543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:09:48.012891018Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:09:48.012919964Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:09:48.012954035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:09:48.020294836Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T06:09:48.276441668Z caller=checkpoint.go:378 msg="writing series" size="329 kB"
level=info ts=2026-02-16T06:09:48.458083446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="462 B" avg_comp="462 B" total_uncomp="345 B" avg_uncomp="345 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:09:48.463243561Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T06:09:48.521906904Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:09:48.679357833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T06:09:48.687978223Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c65066aed%3A19c650697d1%3Af3ec1025?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-02-16T06:09:48.688006577Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:09:48.753733043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:09:48.758138305Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T06:09:48.962808945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:09:48.966935213Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:09:49.064354386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:09:49.068581837Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:09:49.068608995Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:09:49.288086865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T06:09:49.292624335Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c65066aed%3A19c650697d1%3Af3ec1025?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T06:09:49.588676055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:09:49.593819443Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T06:09:51.943196662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:09:51.948540251Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T06:09:52.702151155Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-02-16T06:09:53.520930081Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:09:53.632289823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:09:53.636235406Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T06:09:53.652240355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:09:53.656228569Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:09:53.656257772Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:09:53.656300047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:09:53.660064165Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:09:54.008252508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:09:54.011852361Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:09:54.193295347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:09:54.196913783Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:09:57.127980958Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-02-16T06:09:58.521677083Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:09:59.872674363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:09:59.877113789Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T06:10:00.397037894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T06:10:01.535810853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:10:01.539613078Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:10:01.554722239Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-02-16T06:10:01.557831226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:10:01.561398725Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:10:02.837865796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T06:10:02.842117574Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c65066aed%3A19c650697d1%3Af3ec1025?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T06:10:03.521556292Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T06:10:03.521547334Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:10:03.521576281Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:10:03.521584463Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T06:10:04.088663054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:10:04.092387124Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:10:04.388295348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="928 B" avg_comp="928 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:10:04.3919893Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T06:10:05.74820136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:10:05.980705745Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=info ts=2026-02-16T06:10:06.07600866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:10:06.079835103Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T06:10:08.521681009Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:10:10.318478688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:10:10.321876599Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:10:10.407283985Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=info ts=2026-02-16T06:10:12.397380801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:10:12.400773724Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c65062bfa%3A19c65062bfb%3A13a6e500?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T06:10:13.246958173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:10:13.250524637Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T06:10:13.521101576Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:10:14.833582267Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=warn ts=2026-02-16T06:10:18.521722074Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:10:19.25985463Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=debug ts=2026-02-16T06:10:20.10128641Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:10:21.272303913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:10:21.276353866Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:10:21.276390567Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:10:21.276411418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:10:21.279724871Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:10:21.513701427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:10:21.517196894Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T06:10:21.668491656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:10:21.672038448Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:10:22.837798277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:10:22.841320145Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T06:10:23.521517898Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:10:23.68603951Z caller=checkpoint.go:378 msg="writing series" size="361 kB"
level=debug ts=2026-02-16T06:10:23.686066541Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=10
level=info ts=2026-02-16T06:10:26.258017088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:10:26.261480416Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:10:26.964715364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:10:26.968712624Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T06:10:28.112321811Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=warn ts=2026-02-16T06:10:28.520978669Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:10:32.538705322Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=info ts=2026-02-16T06:10:32.9345192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:10:32.940811198Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T06:10:33.406148903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:10:33.413342698Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:10:33.503213609Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:10:33.504280862Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T06:10:33.521144697Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:10:33.521171153Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:10:33.521195183Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:10:33.521206328Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T06:10:34.137302435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:10:34.141333326Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:10:34.141374543Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:10:34.141404564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:10:34.145391292Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:10:35.670440057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:10:35.674269042Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-02-16T06:10:35.750315347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=info ts=2026-02-16T06:10:36.243732353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:10:36.249921945Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:10:36.82950622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:10:36.833551037Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T06:10:36.964390568Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
ts=2026-02-16T06:10:38.057385057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:50214"
level=warn ts=2026-02-16T06:10:38.520929505Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:10:38.677264045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T06:10:38.682683096Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c65066aed%3A19c650697d1%3Af3ec1025?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T06:10:40.944508271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:10:40.949392266Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T06:10:41.39062678Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=info ts=2026-02-16T06:10:43.110080167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:10:43.114037449Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:10:43.371599346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="462 B" avg_comp="462 B" total_uncomp="345 B" avg_uncomp="345 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:10:43.377078658Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T06:10:43.521682891Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:10:44.904918268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:10:44.909013016Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T06:10:45.816436923Z caller=checkpoint.go:378 msg="writing series" size="869 B"
level=info ts=2026-02-16T06:10:48.22609499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:10:48.229656059Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T06:10:48.521285342Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:10:50.243597004Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T06:10:50.556694031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:10:50.560284547Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:10:52.951627785Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T06:10:53.5209023Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:10:54.669797013Z caller=checkpoint.go:378 msg="writing series" size="414 kB"
level=debug ts=2026-02-16T06:10:55.865919276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T06:10:58.521809388Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:10:59.095526181Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=info ts=2026-02-16T06:10:59.441504578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:10:59.445521015Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:11:00.861591663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:11:00.865251269Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c65062bfa%3A19c65062bfb%3A13a6e500?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
ts=2026-02-16T06:11:01.175535477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:51232"
level=info ts=2026-02-16T06:11:02.945246924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:11:02.949779866Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T06:11:03.521741158Z caller=checkpoint.go:390 msg="flushing series" totalSize="683 kB" series=8
level=warn ts=2026-02-16T06:11:03.521746756Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:11:03.521764204Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:11:03.521782683Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:11:03.521790983Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T06:11:03.585678199Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-02-16T06:11:03.587488441Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.06623338s
ts=2026-02-16T06:11:05.752788776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:11:06.159557026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:48776"
level=info ts=2026-02-16T06:11:06.609128419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:11:06.612929143Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:11:07.393788073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:11:07.397482404Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T06:11:08.521242689Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:11:08.786476661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:11:08.790019392Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:11:09.037770739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:11:09.041021515Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:11:09.615651676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:11:09.62432971Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:11:10.066489245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:11:10.069754319Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T06:11:11.580297533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:11:11.583674027Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T06:11:13.521220176Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:11:15.850955954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:11:15.854562987Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T06:11:17.407991372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:11:17.411632007Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T06:11:18.521488827Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:11:21.102191817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T06:11:21.106271476Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c65066aed%3A19c650697d1%3Af3ec1025?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T06:11:21.416483226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" idle=1 full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:11:21.42001785Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T06:11:23.521111893Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:11:24.407981546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:11:24.4116357Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:11:24.534035515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:11:24.537605512Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:11:24.609829275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="444 B" avg_comp="444 B" total_uncomp="345 B" avg_uncomp="345 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:11:24.613137226Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:11:25.230858489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:11:25.23706632Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:11:25.327506598Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:11:27.734083554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:11:27.737714039Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T06:11:28.521072545Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T06:11:31.177476399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:47282"
level=info ts=2026-02-16T06:11:33.503683895Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T06:11:33.503683794Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:11:33.52074183Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T06:11:33.520864888Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=warn ts=2026-02-16T06:11:33.520907275Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:11:33.520913941Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:11:33.520933701Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:11:33.520941992Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:11:33.52107223Z caller=checkpoint.go:378 msg="writing series" size="138 kB"
level=info ts=2026-02-16T06:11:34.529481945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.4 kB" avg_comp="2.4 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:11:34.534844408Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
ts=2026-02-16T06:11:35.75474493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=debug ts=2026-02-16T06:11:37.378269463Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
ts=2026-02-16T06:11:38.060227263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:48962"
level=warn ts=2026-02-16T06:11:38.521653601Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:11:40.767211643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:11:40.7703438Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T06:11:41.2360138Z caller=checkpoint.go:378 msg="writing series" size="323 kB"
level=debug ts=2026-02-16T06:11:42.148063161Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T06:11:43.520872054Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:11:45.093783782Z caller=checkpoint.go:378 msg="writing series" size="440 kB"
level=info ts=2026-02-16T06:11:47.028979116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:11:47.032326139Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c65062bfa%3A19c65062bfb%3A13a6e500?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T06:11:48.032974107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:11:48.036298667Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T06:11:48.521814105Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:11:48.949792534Z caller=checkpoint.go:378 msg="writing series" size="120 kB"
level=info ts=2026-02-16T06:11:50.11705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:11:50.120318945Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:11:52.239470945Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:11:52.699614736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:11:52.703271106Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:11:52.807486621Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=debug ts=2026-02-16T06:11:52.807505119Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-02-16T06:11:53.521590516Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:11:54.030045309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.3 kB" total_uncomp="55 kB" avg_uncomp="27 kB" idle=1 full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:11:54.033779867Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T06:11:56.664989148Z caller=checkpoint.go:378 msg="writing series" size="257 kB"
level=warn ts=2026-02-16T06:11:58.521041223Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:11:59.116083651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:11:59.119576321Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T06:12:00.521357009Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=info ts=2026-02-16T06:12:02.415347445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:12:02.419059521Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:12:02.475127238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:12:02.47872232Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T06:12:02.858642077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:12:02.861939316Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:12:03.521663374Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:12:03.52169316Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:12:03.521703563Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:12:03.521696812Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:12:04.379071443Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-02-16T06:12:04.563721177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T06:12:04.568073649Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c65066aed%3A19c650697d1%3Af3ec1025?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-02-16T06:12:05.756325524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=info ts=2026-02-16T06:12:05.881207786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:12:05.884598854Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:12:06.751263758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:12:06.75451872Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:12:07.349112215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:12:07.35243041Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T06:12:08.235647605Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-02-16T06:12:08.521174917Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:12:08.901662077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:12:08.904755798Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T06:12:12.092767693Z caller=checkpoint.go:378 msg="writing series" size="221 kB"
level=info ts=2026-02-16T06:12:13.228638771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:12:13.232486081Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T06:12:13.52106665Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:12:15.950303942Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=info ts=2026-02-16T06:12:16.357478126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:12:16.360769196Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:12:18.019104915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="462 B" avg_comp="462 B" total_uncomp="345 B" avg_uncomp="345 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:12:18.023005302Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:12:18.023024045Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:12:18.023043327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="462 B" avg_comp="462 B" total_uncomp="345 B" avg_uncomp="345 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:12:18.026202557Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T06:12:18.52181526Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:12:19.807112915Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T06:12:22.299642889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:12:22.30607379Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:12:22.306094245Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:12:22.306122916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:12:22.311779097Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:12:22.447153497Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T06:12:23.521066751Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:12:23.664424279Z caller=checkpoint.go:378 msg="writing series" size="4.4 kB"
level=info ts=2026-02-16T06:12:23.97547851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:12:23.978819131Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:12:24.206639162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:12:24.210069262Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:12:24.210089478Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:12:24.210111434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:12:24.212975866Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:12:27.521520002Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-02-16T06:12:28.521823705Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:12:28.903032523Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:12:28.904614782Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T06:12:29.774025479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:12:29.777238695Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-02-16T06:12:31.182002191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:40594"
level=debug ts=2026-02-16T06:12:31.378837675Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=info ts=2026-02-16T06:12:31.984142591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:12:31.987569968Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c65062bfa%3A19c65062bfb%3A13a6e500?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T06:12:32.661374073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:12:32.664625173Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T06:12:33.503014032Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:12:33.504141201Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:12:33.521327834Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:12:33.521348356Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:12:33.521356418Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:12:33.521357032Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:12:34.472809239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:12:34.47940939Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:12:35.235543668Z caller=checkpoint.go:378 msg="writing series" size="63 kB"
ts=2026-02-16T06:12:35.757730884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=info ts=2026-02-16T06:12:37.870697683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="462 B" avg_comp="462 B" total_uncomp="345 B" avg_uncomp="345 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:12:37.874407709Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T06:12:38.520993786Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:12:39.093052942Z caller=checkpoint.go:378 msg="writing series" size="88 kB"
level=debug ts=2026-02-16T06:12:42.949665367Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-02-16T06:12:43.028093138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:12:43.032273782Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T06:12:43.520925905Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:12:43.950023506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.3 kB" total_uncomp="55 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:12:43.953527879Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T06:12:46.099198119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:12:46.102495999Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T06:12:46.807591997Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
level=info ts=2026-02-16T06:12:47.571638929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:12:47.575202153Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:12:48.085487347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:12:48.089131283Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T06:12:48.521408986Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:12:48.598697821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:12:48.602122509Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:12:49.77777419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:12:49.781730573Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T06:12:50.664127345Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T06:12:51.125480667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:12:51.129079193Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T06:12:53.520907166Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:12:54.522043807Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=debug ts=2026-02-16T06:12:54.724863103Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T06:12:58.378621145Z caller=checkpoint.go:378 msg="writing series" size="172 kB"
level=info ts=2026-02-16T06:12:58.451871386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T06:12:58.456579965Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c65066aed%3A19c650697d1%3Af3ec1025?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-02-16T06:12:58.521587955Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:12:59.137068869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:12:59.140218606Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:12:59.418010828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="462 B" avg_comp="462 B" total_uncomp="345 B" avg_uncomp="345 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:12:59.421275183Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
ts=2026-02-16T06:13:01.184067389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:55130"
level=info ts=2026-02-16T06:13:01.418942941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:13:01.422148382Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:13:01.716068755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:13:01.719578268Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T06:13:02.23592353Z caller=checkpoint.go:378 msg="writing series" size="175 kB"
level=debug ts=2026-02-16T06:13:02.235946726Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=18
level=info ts=2026-02-16T06:13:03.27006324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:13:03.273894889Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T06:13:03.521060304Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:13:03.521087176Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:13:03.521111241Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:13:03.521119558Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-02-16T06:13:05.759614972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=debug ts=2026-02-16T06:13:06.093337678Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-02-16T06:13:06.093382412Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=1
ts=2026-02-16T06:13:06.167785514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:42588"
level=info ts=2026-02-16T06:13:06.772222861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:13:06.776593632Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:13:08.121778569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:13:08.12527193Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T06:13:08.521491066Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:13:09.342544421Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:13:09.950091117Z caller=checkpoint.go:378 msg="writing series" size="62 kB"
level=info ts=2026-02-16T06:13:10.507663895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:13:10.51103725Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T06:13:13.520916465Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:13:13.807676837Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=info ts=2026-02-16T06:13:13.877823973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:13:13.884812201Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:13:17.664827861Z caller=checkpoint.go:378 msg="writing series" size="63 kB"
level=info ts=2026-02-16T06:13:17.719245886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:13:17.72260969Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T06:13:18.521166831Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:13:20.370425193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:13:20.374489919Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c65062bfa%3A19c65062bfb%3A13a6e500?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T06:13:21.521407226Z caller=checkpoint.go:378 msg="writing series" size="98 kB"
level=warn ts=2026-02-16T06:13:23.521502125Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:13:25.379141272Z caller=checkpoint.go:378 msg="writing series" size="618 kB"
level=info ts=2026-02-16T06:13:27.680000621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:13:27.683681577Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:13:27.983559813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="571 B" avg_comp="571 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:13:27.986946548Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T06:13:28.521924991Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:13:29.236382516Z caller=checkpoint.go:378 msg="writing series" size="37 kB"
level=debug ts=2026-02-16T06:13:29.972909176Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T06:13:31.185877728Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:53076"
level=debug ts=2026-02-16T06:13:33.092697814Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=info ts=2026-02-16T06:13:33.401578241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:13:33.405353249Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:13:33.503478222Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:13:33.504564882Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:13:33.520924735Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:13:33.520950912Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:13:33.520959754Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:13:33.520949541Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:13:33.76447686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:13:33.76841312Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:13:34.746285718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:13:34.749828829Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-02-16T06:13:35.762113205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=info ts=2026-02-16T06:13:36.214186692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.3 kB" total_uncomp="55 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:13:36.218103745Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T06:13:36.950303988Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=warn ts=2026-02-16T06:13:38.521503374Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:13:39.175088051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:13:39.180524757Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:13:40.50732795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="462 B" avg_comp="462 B" total_uncomp="345 B" avg_uncomp="345 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:13:40.511493459Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T06:13:40.807155787Z caller=checkpoint.go:378 msg="writing series" size="939 B"
level=info ts=2026-02-16T06:13:41.586771141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:13:41.589919181Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:13:42.529823884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:13:42.533238172Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T06:13:43.521751681Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:13:43.878315736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:13:43.882273609Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:13:44.664773396Z caller=checkpoint.go:378 msg="writing series" size="102 kB"
level=info ts=2026-02-16T06:13:45.230295652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" idle=1 full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:13:45.234214773Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T06:13:48.521103735Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:13:48.521183462Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=debug ts=2026-02-16T06:13:48.521207841Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-02-16T06:13:50.857537835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:13:50.861147502Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T06:13:51.306446514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:13:51.310253489Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:13:52.290241629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:13:52.294032786Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:13:52.385881151Z caller=checkpoint.go:378 msg="writing series" size="12 MB"
level=debug ts=2026-02-16T06:13:52.385906089Z caller=checkpoint.go:390 msg="flushing series" totalSize="12 MB" series=1
level=warn ts=2026-02-16T06:13:53.521256908Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:13:55.326306284Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:13:56.130655389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:13:56.134433941Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:13:56.235667712Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-02-16T06:13:56.64980369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:13:56.653536935Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:13:58.338489169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T06:13:58.343011386Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c65066aed%3A19c650697d1%3Af3ec1025?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-02-16T06:13:58.494231685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T06:13:58.521017256Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:14:00.092698006Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T06:14:03.521205428Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:14:03.521244428Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:14:03.521253497Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:14:03.52125377Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:14:03.953186588Z caller=checkpoint.go:378 msg="writing series" size="5.1 MB"
level=debug ts=2026-02-16T06:14:03.953219994Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.1 MB" series=3
ts=2026-02-16T06:14:05.764179322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:14:07.807872905Z caller=checkpoint.go:378 msg="writing series" size="379 kB"
level=info ts=2026-02-16T06:14:08.421346102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:14:08.426129166Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c65062bfa%3A19c65062bfb%3A13a6e500?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T06:14:08.439176389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:14:08.446480788Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T06:14:08.521553587Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:14:11.664604555Z caller=checkpoint.go:378 msg="writing series" size="512 B"
level=info ts=2026-02-16T06:14:13.118644303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:14:13.124314973Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T06:14:13.521033494Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:14:14.0543785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:14:14.058473568Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:14:14.771476865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="323 B" avg_comp="323 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:14:14.775592157Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:14:15.521803235Z caller=checkpoint.go:378 msg="writing series" size="6.4 kB"
level=warn ts=2026-02-16T06:14:18.521393941Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:14:19.378418063Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=debug ts=2026-02-16T06:14:23.236013615Z caller=checkpoint.go:378 msg="writing series" size="125 kB"
level=warn ts=2026-02-16T06:14:23.52181186Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:14:23.690089339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:14:23.694303569Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:14:23.850787375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:14:23.854341243Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:14:26.222727445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:14:26.226670746Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T06:14:27.092534066Z caller=checkpoint.go:378 msg="writing series" size="161 kB"
level=info ts=2026-02-16T06:14:28.164229537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:14:28.167619795Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T06:14:28.521161206Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:14:29.46159436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:14:29.465428753Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T06:14:29.506785662Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:14:29.506827239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="35 kB" avg_comp="35 kB" total_uncomp="35 kB" avg_uncomp="35 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:14:29.5097297Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T06:14:30.950459019Z caller=checkpoint.go:378 msg="writing series" size="145 kB"
level=debug ts=2026-02-16T06:14:33.503045785Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:14:33.504090526Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:14:33.521805453Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T06:14:33.521805338Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:14:33.521826416Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:14:33.521834329Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T06:14:33.850160512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.3 kB" total_uncomp="55 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:14:33.853433669Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T06:14:34.807537992Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
ts=2026-02-16T06:14:35.766813436Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=info ts=2026-02-16T06:14:35.830075839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:14:35.833508548Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:14:37.637980029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:14:37.641669297Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:14:38.368110736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="462 B" avg_comp="462 B" total_uncomp="345 B" avg_uncomp="345 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:14:38.371848674Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T06:14:38.52089045Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:14:38.664180709Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T06:14:39.580220303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:14:39.584191529Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T06:14:41.95830093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:14:41.961623647Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T06:14:42.087961532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T06:14:42.265412097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:14:42.268623814Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T06:14:42.521571569Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T06:14:43.25869342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="622 B" avg_comp="622 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:14:43.262931044Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T06:14:43.521388313Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:14:43.958626877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:14:43.961979708Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T06:14:43.961999692Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:14:43.962020398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:14:43.965155941Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:14:46.378504588Z caller=checkpoint.go:378 msg="writing series" size="463 kB"
level=debug ts=2026-02-16T06:14:46.378531295Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=11
level=info ts=2026-02-16T06:14:47.672959979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T06:14:47.676786352Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c65066aed%3A19c650697d1%3Af3ec1025?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T06:14:47.790089935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:14:47.793624637Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T06:14:48.521781024Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:14:50.236049053Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=warn ts=2026-02-16T06:14:53.521784891Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:14:53.960160695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:14:53.963830457Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:14:54.093044327Z caller=checkpoint.go:378 msg="writing series" size="358 kB"
level=info ts=2026-02-16T06:14:56.66411579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:14:56.667700922Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:14:57.950578569Z caller=checkpoint.go:378 msg="writing series" size="298 kB"
level=info ts=2026-02-16T06:14:58.310479749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:14:58.317403165Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T06:14:58.521488366Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:14:58.881592899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:14:58.885977746Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c65062bfa%3A19c65062bfb%3A13a6e500?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T06:15:00.939714145Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:15:00.939789758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:15:00.94468664Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T06:15:01.807323101Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T06:15:03.252637797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:15:03.256266799Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:15:03.521004684Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:15:03.521034792Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:15:03.521043476Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:15:03.52109928Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:15:04.641411034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:15:04.644494423Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T06:15:05.66421886Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
ts=2026-02-16T06:15:05.768841719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=info ts=2026-02-16T06:15:07.786605182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:15:07.790292278Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:15:07.89761847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:15:07.901222185Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T06:15:08.521415918Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:15:09.521243535Z caller=checkpoint.go:378 msg="writing series" size="135 kB"
level=info ts=2026-02-16T06:15:10.134881545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:15:10.139513361Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:15:12.16996587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:15:12.175404367Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T06:15:13.378450459Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=warn ts=2026-02-16T06:15:13.521812996Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:15:15.057738018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:15:15.061044547Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T06:15:17.236204601Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=warn ts=2026-02-16T06:15:18.521406985Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:15:19.61115915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="928 B" avg_comp="928 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:15:19.614786379Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:15:20.711306361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.3 kB" total_uncomp="55 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:15:20.715086079Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T06:15:21.09333736Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=debug ts=2026-02-16T06:15:21.093355818Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-02-16T06:15:21.14892943Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T06:15:22.699328027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:15:22.702679139Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:15:23.172104894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:15:23.175733765Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T06:15:23.521550399Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:15:24.835183173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:15:24.838680457Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T06:15:24.950109131Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=info ts=2026-02-16T06:15:26.448965022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:15:26.452355077Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:15:26.527777171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:15:26.53105637Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T06:15:26.531077915Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:15:26.531103626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:15:26.534117235Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:15:27.455486319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="462 B" avg_comp="462 B" total_uncomp="345 B" avg_uncomp="345 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:15:27.458940968Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T06:15:28.521644815Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:15:28.807577849Z caller=checkpoint.go:378 msg="writing series" size="582 kB"
level=info ts=2026-02-16T06:15:30.074493409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:15:30.078641441Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T06:15:31.192824864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:45612"
level=info ts=2026-02-16T06:15:32.325297452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:15:32.329176534Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:15:32.664095829Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=debug ts=2026-02-16T06:15:33.503433286Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:15:33.504478545Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T06:15:33.504684555Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-02-16T06:15:33.504725036Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1968024 n=1
level=debug ts=2026-02-16T06:15:33.504771308Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1968024 n=1
level=info ts=2026-02-16T06:15:33.52128178Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:15:33.521302028Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:15:33.521310286Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:15:33.521311609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:15:33.605806947Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:15:33.635856778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:15:33.639239727Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
ts=2026-02-16T06:15:35.770058501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=info ts=2026-02-16T06:15:36.25965464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T06:15:36.263357097Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c65066aed%3A19c650697d1%3Af3ec1025?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-02-16T06:15:36.521890275Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=info ts=2026-02-16T06:15:37.760107527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:15:37.763770961Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:15:38.008426818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:15:38.014423942Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T06:15:38.521726667Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:15:40.378995413Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-02-16T06:15:40.732089252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:15:40.73617737Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T06:15:40.73619675Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:15:40.736223549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:15:40.739599297Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T06:15:43.521527783Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:15:44.235732655Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-02-16T06:15:44.508848321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:15:44.513064726Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c65062bfa%3A19c65062bfb%3A13a6e500?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-02-16T06:15:44.513092716Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:15:44.51312141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="40 kB" avg_comp="40 kB" total_uncomp="311 kB" avg_uncomp="311 kB" idle=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:15:44.516531011Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T06:15:48.006272235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:15:48.009827791Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:15:48.092957282Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T06:15:48.404991227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:15:48.411315094Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T06:15:48.52182006Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:15:48.988405463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:15:48.991686582Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:15:51.104855369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:15:51.108218132Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:15:51.108240313Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:15:51.108261708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:15:51.111097009Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:15:51.950525043Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-02-16T06:15:53.521098308Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:15:55.225481492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:15:55.229878144Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:15:55.229900077Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:15:55.229921162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:15:55.232677558Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T06:15:55.807616332Z caller=checkpoint.go:378 msg="writing series" size="450 kB"
level=debug ts=2026-02-16T06:15:55.8076382Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=9
level=info ts=2026-02-16T06:15:55.834559907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:15:55.837956691Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T06:15:58.521880508Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:15:59.664585971Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=debug ts=2026-02-16T06:16:00.743522871Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:16:00.745545399Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:16:01.194531798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:50780"
level=info ts=2026-02-16T06:16:02.08021667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:16:02.083502241Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T06:16:02.41563821Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:16:02.955623318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:16:02.958988014Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T06:16:03.521745827Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.8 kB" series=1
level=info ts=2026-02-16T06:16:03.521763359Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:16:03.521787143Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:16:03.521794794Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:16:03.521798615Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:16:03.579452215Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-02-16T06:16:03.583915521Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.062937301s
level=info ts=2026-02-16T06:16:04.226974171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:16:04.230243835Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:16:05.08954637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:16:05.092805101Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:16:05.09282627Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:16:05.092849083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:16:05.095524797Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:16:05.527814056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:16:05.530903897Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
ts=2026-02-16T06:16:05.772113153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:16:06.181247859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:53900"
level=warn ts=2026-02-16T06:16:08.521662759Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:16:08.546796347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:16:08.550228191Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:16:11.585396078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:16:11.590308488Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T06:16:12.886093421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:16:12.889438631Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:16:12.917757873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:16:12.92090757Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:16:12.920926697Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:16:12.9209497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:16:12.923906685Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T06:16:13.521943243Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:16:13.736112548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:16:13.739774384Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:16:13.739793777Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:16:13.739813708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:16:13.742813758Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:16:14.300160997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="570 B" avg_comp="285 B" total_uncomp="689 B" avg_uncomp="344 B" idle=1 full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:16:14.304097857Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:16:14.870492914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:16:14.874424888Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:16:14.874445797Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:16:14.874468173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:16:14.877550332Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:16:15.226669629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:16:15.23033457Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:16:15.230357665Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:16:15.230402677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:16:15.233265155Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T06:16:17.053566768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.3 kB" total_uncomp="55 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:16:17.05734262Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:16:17.057376623Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:16:17.057399914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.3 kB" total_uncomp="55 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:16:17.060627684Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T06:16:17.176020255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T06:16:17.179607677Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c65066aed%3A19c650697d1%3Af3ec1025?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-02-16T06:16:17.179629752Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:16:17.448576098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:16:17.454999696Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:16:17.657430702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="324 B" avg_comp="324 B" total_uncomp="96 kB" avg_uncomp="96 kB" idle=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:16:17.660805214Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T06:16:18.521152069Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:16:18.868316922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:16:18.873309171Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:16:19.250208919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:16:19.253477588Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:16:19.253498276Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:16:19.253519393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:16:19.256222048Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:16:21.30053608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:16:21.303658769Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:16:21.303677888Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:16:21.303698073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:16:21.306423046Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T06:16:23.521391067Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:16:25.99411249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:16:25.997597343Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:16:27.234024916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:16:27.237322294Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T06:16:27.565258362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:16:27.568496859Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T06:16:27.698902883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:16:27.702051425Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T06:16:28.521801029Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:16:28.819497209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:16:28.822880004Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:16:29.798695099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:16:29.802173985Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:16:30.412758497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:16:30.41630362Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T06:16:31.608646486Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:16:31.912545775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:16:31.916493286Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:16:32.014688125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:16:32.021011778Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:16:33.502824635Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:16:33.503831786Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:16:33.521320518Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T06:16:33.521331695Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:16:33.521351258Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:16:33.521374606Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:16:33.521381949Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:16:33.521439746Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=debug ts=2026-02-16T06:16:33.521689443Z caller=checkpoint.go:378 msg="writing series" size="200 kB"
ts=2026-02-16T06:16:35.774534141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:16:36.18233519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:56786"
level=info ts=2026-02-16T06:16:36.384308863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.3 kB" total_uncomp="55 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:16:36.387851785Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T06:16:36.771889123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:16:36.775090297Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T06:16:37.324654098Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=info ts=2026-02-16T06:16:37.998934159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:16:38.003516844Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:16:38.317552071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:16:38.32102849Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T06:16:38.521701883Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:16:39.172860547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:16:39.176221447Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T06:16:41.128470829Z caller=checkpoint.go:378 msg="writing series" size="323 kB"
level=info ts=2026-02-16T06:16:41.448230011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:16:41.451150685Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:16:41.451173087Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:16:41.451199532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:16:41.453981054Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T06:16:43.521734302Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:16:43.710190498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:16:44.9306546Z caller=checkpoint.go:378 msg="writing series" size="774 kB"
level=debug ts=2026-02-16T06:16:44.930680652Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-02-16T06:16:47.77649501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:16:47.780449638Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:16:48.515575661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:16:48.519277245Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T06:16:48.521334177Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:16:48.733325237Z caller=checkpoint.go:378 msg="writing series" size="194 kB"
level=info ts=2026-02-16T06:16:49.751615556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:16:49.755279327Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:16:50.39803662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:16:50.40152113Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:16:50.401540683Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:16:50.401570998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:16:50.404510313Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:16:51.573401875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:16:51.576904018Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:16:52.054149498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:16:52.057510374Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T06:16:52.535891553Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T06:16:53.162270474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:16:53.165717917Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T06:16:53.520925726Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:16:56.339458951Z caller=checkpoint.go:378 msg="writing series" size="166 kB"
level=info ts=2026-02-16T06:16:57.607016498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="12 kB" avg_comp="5.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:16:57.6106338Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:16:57.610652206Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:16:57.610672795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="12 kB" avg_comp="5.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:16:57.61330625Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:16:57.703629804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.3 kB" total_uncomp="55 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:16:57.70697545Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T06:16:57.954542811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:16:57.957815229Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:16:58.062039515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:16:58.066110112Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T06:16:58.52100335Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:16:59.49539318Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T06:17:00.142304997Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
ts=2026-02-16T06:17:01.198285245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:42794"
level=info ts=2026-02-16T06:17:01.441214435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:17:01.446302546Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:17:03.521170588Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T06:17:03.521178512Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:17:03.521192086Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:17:03.521200159Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:17:03.94468109Z caller=checkpoint.go:378 msg="writing series" size="127 kB"
level=info ts=2026-02-16T06:17:05.663975282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:17:05.668646494Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T06:17:05.776595711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=info ts=2026-02-16T06:17:07.214415332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="588 B" avg_comp="294 B" total_uncomp="689 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:17:07.217731253Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:17:07.384132031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:17:07.387328727Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T06:17:07.686007781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:17:07.689320898Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T06:17:07.747721151Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-02-16T06:17:08.521582753Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:17:08.531725415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="12 kB" avg_comp="5.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:17:08.535129027Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:17:09.944844322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:17:09.948382141Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T06:17:11.550473369Z caller=checkpoint.go:378 msg="writing series" size="334 kB"
level=warn ts=2026-02-16T06:17:13.521777607Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:17:14.459637154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:17:14.464544964Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T06:17:15.353040784Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=info ts=2026-02-16T06:17:15.47929172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:17:15.483037205Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T06:17:18.520895055Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:17:18.816113981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:17:18.820390298Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:17:19.156317051Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T06:17:21.239985936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:17:21.2435071Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T06:17:22.24747914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:17:22.251290801Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:17:22.594987966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.4 kB" avg_comp="2.4 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:17:22.598453499Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:17:22.958962071Z caller=checkpoint.go:378 msg="writing series" size="4.4 kB"
level=warn ts=2026-02-16T06:17:23.521672229Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:17:25.703802536Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T06:17:26.761310684Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
level=warn ts=2026-02-16T06:17:28.521756853Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:17:28.684125348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:17:28.687819177Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:17:29.265995803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:17:29.272451046Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:17:30.564795439Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=info ts=2026-02-16T06:17:31.728630355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:17:31.731922214Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T06:17:32.710759883Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:17:33.503322567Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:17:33.504384973Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:17:33.52096267Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T06:17:33.520964681Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:17:33.520981473Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:17:33.520989826Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T06:17:33.712285001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:17:33.715636268Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:17:34.367336977Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=info ts=2026-02-16T06:17:35.374969029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:17:35.378265378Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T06:17:35.778563829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:17:38.170514007Z caller=checkpoint.go:378 msg="writing series" size="332 kB"
level=debug ts=2026-02-16T06:17:38.170536013Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=14
level=warn ts=2026-02-16T06:17:38.521901001Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:17:38.848006075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:17:38.853616635Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T06:17:41.972784404Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T06:17:42.276487669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.3 kB" total_uncomp="55 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:17:42.280125749Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T06:17:42.381503207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:17:42.385000353Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:17:43.454204495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="12 kB" avg_comp="5.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:17:43.457671819Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T06:17:43.520883527Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:17:45.776014652Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
level=info ts=2026-02-16T06:17:46.397021595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:17:46.40354953Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:17:46.418601966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:17:46.422186868Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:17:47.423082058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:17:47.426499846Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:17:47.429567121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:17:47.43260078Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T06:17:48.521514338Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:17:49.578925398Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=debug ts=2026-02-16T06:17:53.381436265Z caller=checkpoint.go:378 msg="writing series" size="42 kB"
level=warn ts=2026-02-16T06:17:53.521071482Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:17:54.635021568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:17:54.640149791Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T06:17:57.18425465Z caller=checkpoint.go:378 msg="writing series" size="258 kB"
level=info ts=2026-02-16T06:17:57.521527181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:17:57.527581225Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T06:17:58.520906902Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:17:59.450631071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:17:59.454877411Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T06:18:00.449837426Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T06:18:00.987421884Z caller=checkpoint.go:378 msg="writing series" size="232 kB"
level=info ts=2026-02-16T06:18:01.983826545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="588 B" avg_comp="294 B" total_uncomp="689 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:18:01.987869888Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:18:03.521801963Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:18:03.521833596Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T06:18:03.521832551Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:18:03.521845031Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T06:18:04.645974123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:18:04.649659414Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T06:18:04.733009587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:18:04.736831073Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T06:18:04.790806187Z caller=checkpoint.go:378 msg="writing series" size="1.7 MB"
level=debug ts=2026-02-16T06:18:04.790825881Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.3 MB" series=7
level=info ts=2026-02-16T06:18:05.438651263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:18:05.441885844Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T06:18:05.780494493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=info ts=2026-02-16T06:18:08.09705022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:18:08.100621149Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T06:18:08.52151177Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:18:08.592881778Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=debug ts=2026-02-16T06:18:10.955616849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:18:12.395118703Z caller=checkpoint.go:378 msg="writing series" size="6.1 kB"
level=warn ts=2026-02-16T06:18:13.521247638Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:18:14.253418907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:18:14.25721645Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T06:18:16.198494683Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
level=info ts=2026-02-16T06:18:16.42216584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:18:16.431149159Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:18:18.240181665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:18:18.243900044Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T06:18:18.521670968Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:18:19.002978502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="571 B" avg_comp="571 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:18:19.006412492Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T06:18:20.001148016Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=info ts=2026-02-16T06:18:21.732331834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:18:21.736087953Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T06:18:23.520911269Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:18:23.803876991Z caller=checkpoint.go:378 msg="writing series" size="399 kB"
level=info ts=2026-02-16T06:18:26.610875894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:18:26.615198482Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T06:18:27.607449139Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-02-16T06:18:27.607480842Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.3 MB" series=6
level=warn ts=2026-02-16T06:18:28.521049735Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:18:30.438787317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:18:30.4424461Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:18:31.243769729Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T06:18:31.409318922Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=info ts=2026-02-16T06:18:33.503744721Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T06:18:33.504033685Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:18:33.521531029Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:18:33.521561687Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:18:33.521570807Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:18:33.521579822Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:18:35.212346319Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
ts=2026-02-16T06:18:35.782207854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=info ts=2026-02-16T06:18:37.441890588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.3 kB" total_uncomp="55 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:18:37.445757356Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-02-16T06:18:38.521730103Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:18:38.803708533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:18:38.807455239Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:18:38.923065598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:18:38.927633024Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:18:39.004561354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:18:39.008508443Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T06:18:39.015591583Z caller=checkpoint.go:378 msg="writing series" size="939 B"
level=info ts=2026-02-16T06:18:40.335237317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:18:40.338585002Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T06:18:42.818291349Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=info ts=2026-02-16T06:18:42.968822505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="12 kB" avg_comp="4.0 kB" total_uncomp="114 kB" avg_uncomp="38 kB" idle=1 full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:18:42.972930377Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:18:43.199671132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:18:43.203614797Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T06:18:43.521419395Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:18:44.056875146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:18:44.062485793Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T06:18:46.620770773Z caller=checkpoint.go:378 msg="writing series" size="144 kB"
level=info ts=2026-02-16T06:18:48.31125875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:18:48.318286696Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T06:18:48.423509448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:18:48.427893493Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T06:18:48.521191149Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:18:50.437518262Z caller=checkpoint.go:378 msg="writing series" size="19 MB"
level=debug ts=2026-02-16T06:18:50.437544972Z caller=checkpoint.go:390 msg="flushing series" totalSize="19 MB" series=6
level=info ts=2026-02-16T06:18:52.511480671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:18:52.516630709Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T06:18:53.521842488Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:18:54.088250156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="588 B" avg_comp="294 B" total_uncomp="689 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:18:54.095611056Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:18:54.095637442Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:18:54.095665403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="588 B" avg_comp="294 B" total_uncomp="689 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:18:54.098639667Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T06:18:54.227601647Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=debug ts=2026-02-16T06:18:58.029665628Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T06:18:58.219440952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:18:58.226198669Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T06:18:58.521792316Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:18:59.199628152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:18:59.203818031Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:19:01.365083995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:19:01.368949643Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:19:01.837186698Z caller=checkpoint.go:378 msg="writing series" size="5.4 MB"
level=debug ts=2026-02-16T06:19:01.837211469Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.4 MB" series=3
level=info ts=2026-02-16T06:19:02.733776406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:19:02.737536313Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T06:19:03.401629823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:19:03.405121089Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T06:19:03.520882639Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:19:03.520921994Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:19:03.520935895Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:19:03.520935388Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:19:04.735874689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:19:05.635397573Z caller=checkpoint.go:378 msg="writing series" size="40 kB"
ts=2026-02-16T06:19:05.783419518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=debug ts=2026-02-16T06:19:06.02308939Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:19:07.410923785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:19:07.415086474Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T06:19:08.521825089Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:19:09.049082996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:19:09.052596901Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:19:09.437802712Z caller=checkpoint.go:378 msg="writing series" size="260 kB"
level=info ts=2026-02-16T06:19:11.465155538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:19:11.468956278Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:19:11.517399907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="588 B" avg_comp="294 B" total_uncomp="689 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:19:11.521388341Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T06:19:13.240808398Z caller=checkpoint.go:378 msg="writing series" size="512 B"
level=warn ts=2026-02-16T06:19:13.521802939Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:19:14.76925347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:19:14.775683799Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:19:14.775701939Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:19:14.775723967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:19:14.782556406Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:19:17.043344362Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-02-16T06:19:17.867584039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:19:17.871286573Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:19:17.871306541Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:19:17.871330011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:19:17.874335089Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T06:19:18.52117107Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:19:20.846376106Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T06:19:20.987881757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:19:20.992538046Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:19:21.651485578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" idle=1 full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:19:21.654727149Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:19:22.044796048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:19:22.047906417Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:19:22.865502128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.3 kB" total_uncomp="55 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:19:22.869033645Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-02-16T06:19:23.521771283Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:19:24.649072732Z caller=checkpoint.go:378 msg="writing series" size="176 kB"
level=debug ts=2026-02-16T06:19:28.451609785Z caller=checkpoint.go:378 msg="writing series" size="308 kB"
level=warn ts=2026-02-16T06:19:28.521809449Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:19:28.818035053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:19:28.823871803Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:19:29.559074116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:19:29.56416567Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-02-16T06:19:31.206771931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:51850"
level=info ts=2026-02-16T06:19:31.235025597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="12 kB" avg_comp="4.0 kB" total_uncomp="114 kB" avg_uncomp="38 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:19:31.238506953Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T06:19:32.254752505Z caller=checkpoint.go:378 msg="writing series" size="259 kB"
level=debug ts=2026-02-16T06:19:32.254774502Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-02-16T06:19:33.50310584Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:19:33.504156063Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:19:33.520901849Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:19:33.520931753Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:19:33.520941871Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:19:33.520953468Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:19:33.805845929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:19:33.809823329Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:19:34.223216033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:19:34.230694792Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:19:34.567654919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:19:34.571090445Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:19:34.84886421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:19:34.852507043Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T06:19:34.979994298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="588 B" avg_comp="294 B" total_uncomp="689 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:19:34.983251256Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:19:35.104647182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:19:35.108174414Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-02-16T06:19:35.785684136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=debug ts=2026-02-16T06:19:36.057409821Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=debug ts=2026-02-16T06:19:36.689213366Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:19:36.930071714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="622 B" avg_comp="622 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:19:36.933673074Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:19:37.003907355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:19:37.007447943Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T06:19:38.521435466Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:19:39.860521509Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T06:19:43.521713826Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:19:43.663073838Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T06:19:44.608783186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:19:44.613245052Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T06:19:45.074807824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="1.9 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:19:45.078506114Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T06:19:47.466274696Z caller=checkpoint.go:378 msg="writing series" size="474 kB"
level=warn ts=2026-02-16T06:19:48.521633075Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:19:51.106894024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:19:51.111766893Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:19:51.269062671Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-02-16T06:19:53.406356419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:19:53.410298822Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T06:19:53.521629547Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:19:55.071399034Z caller=checkpoint.go:378 msg="writing series" size="780 kB"
level=debug ts=2026-02-16T06:19:55.071425422Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-02-16T06:19:55.416277045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="323 B" avg_comp="323 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:19:55.420211506Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T06:19:58.521942357Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:19:58.8743948Z caller=checkpoint.go:378 msg="writing series" size="440 kB"
level=info ts=2026-02-16T06:20:01.007423846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:20:01.011483049Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:20:01.418508118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:20:01.422134774Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T06:20:02.036881564Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:20:02.038942281Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T06:20:02.676803707Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T06:20:03.026494121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.3 kB" total_uncomp="55 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:20:03.030396496Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T06:20:03.057406054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:20:03.060577056Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:20:03.478808533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:20:03.482166875Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T06:20:03.521191686Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:20:03.521227299Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:20:03.521239559Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:20:03.521248854Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T06:20:03.721329696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:20:03.727630494Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:20:04.636026698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:20:04.639420203Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T06:20:05.787709988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:20:06.196481325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:58838"
level=debug ts=2026-02-16T06:20:06.479452332Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T06:20:08.521585588Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:20:10.282975832Z caller=checkpoint.go:378 msg="writing series" size="263 kB"
level=debug ts=2026-02-16T06:20:11.556714016Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:20:11.817516487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:20:11.820897875Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T06:20:13.521027912Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:20:14.085735722Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=info ts=2026-02-16T06:20:17.222416665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:20:17.231376682Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T06:20:17.888232183Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=warn ts=2026-02-16T06:20:18.521520118Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:20:19.154598544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:20:19.157829798Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:20:21.040568173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:20:21.044400354Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T06:20:21.691212171Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=info ts=2026-02-16T06:20:22.413310193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:20:22.419899501Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:20:22.864917273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:20:22.868806006Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T06:20:22.98817119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="12 kB" avg_comp="4.0 kB" total_uncomp="114 kB" avg_uncomp="38 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:20:22.991433977Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T06:20:23.521871909Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:20:25.494514513Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=info ts=2026-02-16T06:20:26.686333388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:20:26.690031449Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:20:26.889839358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:20:26.893202857Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:20:28.337614276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:20:28.340843699Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T06:20:28.521778406Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:20:29.297104962Z caller=checkpoint.go:378 msg="writing series" size="700 kB"
level=debug ts=2026-02-16T06:20:29.297128845Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=9
level=debug ts=2026-02-16T06:20:33.100073693Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=info ts=2026-02-16T06:20:33.315882153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:20:33.32018811Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T06:20:33.502623973Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:20:33.503686003Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:20:33.521456541Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:20:33.521473268Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:20:33.521481312Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:20:33.52149931Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:20:34.275725053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="588 B" avg_comp="294 B" total_uncomp="689 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:20:34.279343248Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:20:35.466626009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:20:35.470296059Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:20:35.619885352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="928 B" avg_comp="928 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:20:35.62316838Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T06:20:35.789155635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=info ts=2026-02-16T06:20:36.498293176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:20:36.50191981Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T06:20:36.902144477Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
ts=2026-02-16T06:20:38.09096563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:53340"
level=warn ts=2026-02-16T06:20:38.52095638Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:20:40.573036553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:20:40.576890277Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:20:40.705408398Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=warn ts=2026-02-16T06:20:43.521788935Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:20:43.745615507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:20:43.74921713Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:20:43.908668443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:20:43.912127065Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T06:20:44.507947166Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=debug ts=2026-02-16T06:20:46.04810373Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T06:20:48.310488383Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=warn ts=2026-02-16T06:20:48.521243706Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:20:51.465956133Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:20:52.114046732Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB"
level=info ts=2026-02-16T06:20:52.367701076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:20:52.370839986Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T06:20:52.392906292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:20:52.395786183Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T06:20:53.521432736Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:20:55.916716201Z caller=checkpoint.go:378 msg="writing series" size="522 kB"
level=info ts=2026-02-16T06:20:56.551302287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:20:56.55521212Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:20:56.734750068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:20:56.73798011Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T06:20:58.521826474Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:20:59.240242613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:20:59.246460114Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:20:59.320870131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.5 kB" total_uncomp="55 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:20:59.3240954Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T06:20:59.71931609Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
ts=2026-02-16T06:21:01.213561609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:42940"
level=info ts=2026-02-16T06:21:03.521017778Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T06:21:03.521025285Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:21:03.521042628Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:21:03.521050853Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:21:03.522099385Z caller=checkpoint.go:390 msg="flushing series" totalSize="828 kB" series=8
level=info ts=2026-02-16T06:21:03.526951428Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-02-16T06:21:03.529728945Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.008159479s
ts=2026-02-16T06:21:05.790706394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=info ts=2026-02-16T06:21:07.462506784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:21:07.465998959Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:21:07.46601978Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:21:07.466044038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:21:07.468801255Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T06:21:08.521582823Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:21:09.625407051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:21:09.628885769Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:21:11.515001675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:21:11.518281641Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T06:21:13.520914422Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:21:17.352009496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=info ts=2026-02-16T06:21:17.353153069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:21:17.356003748Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:21:17.356035419Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:21:18.44097802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:21:18.444515932Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T06:21:18.521210681Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:21:19.932378924Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:21:19.932416804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="210 kB" avg_comp="210 kB" total_uncomp="833 kB" avg_uncomp="833 kB" idle=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:21:19.939077942Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=info ts=2026-02-16T06:21:21.228581568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:21:21.234156132Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:21:21.234176108Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:21:21.234202219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:21:21.23890137Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:21:21.487506907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:21:21.490968601Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:21:22.077668722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="12 kB" avg_comp="4.0 kB" total_uncomp="114 kB" avg_uncomp="38 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:21:22.081224322Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:21:22.60870999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:21:22.612311533Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T06:21:23.521758491Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:21:24.089809271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:21:24.0936177Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:21:24.802465434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:21:24.806201652Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:21:25.483856452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:21:25.487300333Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T06:21:27.033897712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:21:27.038556979Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T06:21:28.521497206Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:21:30.359226425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="887 B" avg_comp="443 B" total_uncomp="689 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:21:30.362420741Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:21:31.611400105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:21:31.614461593Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T06:21:32.483330356Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:21:33.503135452Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:21:33.504184611Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:21:33.521642166Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:21:33.521673918Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:21:33.521669Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T06:21:33.521684293Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:21:33.521702771Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:21:33.521788568Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=debug ts=2026-02-16T06:21:33.522053058Z caller=checkpoint.go:378 msg="writing series" size="257 kB"
level=info ts=2026-02-16T06:21:34.577409913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.4 kB" avg_comp="2.4 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:21:34.580553035Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T06:21:34.580571442Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:21:34.580590577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:21:34.583168359Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:21:35.012422188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:21:35.015804992Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
ts=2026-02-16T06:21:35.79212507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:21:37.324992722Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=info ts=2026-02-16T06:21:38.109476934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:21:38.115570201Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T06:21:38.521799762Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:21:39.497694329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:21:39.501389989Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:21:40.273386252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:21:40.277057907Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T06:21:41.128545688Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=info ts=2026-02-16T06:21:42.302435266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:21:42.305933221Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T06:21:43.521257258Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:21:44.930816839Z caller=checkpoint.go:378 msg="writing series" size="738 kB"
level=debug ts=2026-02-16T06:21:44.930843065Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-02-16T06:21:48.521180737Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:21:48.734139355Z caller=checkpoint.go:378 msg="writing series" size="269 kB"
level=debug ts=2026-02-16T06:21:49.022455513Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:21:49.903841144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:21:49.908213433Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T06:21:50.27795563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:21:50.282610055Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:21:51.876232554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.7 kB" total_uncomp="55 kB" avg_uncomp="18 kB" idle=1 full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:21:51.879987046Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T06:21:52.042486477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:21:52.046075024Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:21:52.220652992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:21:52.224303485Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T06:21:52.536389621Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-02-16T06:21:53.521681477Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:21:56.339818966Z caller=checkpoint.go:378 msg="writing series" size="283 kB"
level=info ts=2026-02-16T06:21:57.650300499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:21:57.65714064Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:21:58.466323219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:21:58.470301027Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T06:21:58.52105053Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:21:58.930722912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:21:58.934045538Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:21:58.934063667Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:21:58.934084186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:21:58.936929999Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T06:21:59.286720851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:21:59.291688084Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T06:22:00.142137298Z caller=checkpoint.go:378 msg="writing series" size="2.3 kB"
level=info ts=2026-02-16T06:22:00.602523753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:22:00.605585903Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:22:03.521141688Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:22:03.521168308Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:22:03.521176365Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:22:03.521197557Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:22:03.945196509Z caller=checkpoint.go:378 msg="writing series" size="158 kB"
ts=2026-02-16T06:22:05.793957568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:22:06.204113867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:44184"
level=info ts=2026-02-16T06:22:07.401610587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:22:07.406662667Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=debug ts=2026-02-16T06:22:07.748313089Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
ts=2026-02-16T06:22:08.095995169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:54388"
level=info ts=2026-02-16T06:22:08.410161391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:22:08.413873147Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T06:22:08.521798104Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:22:09.005566832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:22:09.008601167Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:22:09.250822552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:22:09.254562111Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T06:22:09.407125918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:22:09.410756926Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:22:10.493470923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:22:10.496726239Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:22:10.496745821Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:22:10.496766489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:22:10.499407391Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:22:10.859414876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:22:10.862526663Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T06:22:10.862544543Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:22:10.862564805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:22:10.865139393Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:22:11.474953847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:22:11.478118497Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:22:11.550988695Z caller=checkpoint.go:378 msg="writing series" size="394 kB"
level=debug ts=2026-02-16T06:22:11.551006451Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-02-16T06:22:13.521788917Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:22:15.353938502Z caller=checkpoint.go:378 msg="writing series" size="66 kB"
level=info ts=2026-02-16T06:22:17.05256012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="12 kB" avg_comp="4.0 kB" total_uncomp="114 kB" avg_uncomp="38 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:22:17.056197949Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:22:17.443281875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:22:17.44655241Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:22:18.320981711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="905 B" avg_comp="452 B" total_uncomp="689 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:22:18.324521739Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T06:22:18.52173916Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:22:19.15597859Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T06:22:20.614127122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:22:20.618317717Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T06:22:20.891989552Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:22:21.622973006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:22:21.626491785Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:22:22.006508858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:22:22.010387943Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:22:22.010407358Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:22:22.010426988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:22:22.013244177Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:22:22.660965561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:22:22.664894959Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:22:22.959068119Z caller=checkpoint.go:378 msg="writing series" size="4.4 kB"
level=warn ts=2026-02-16T06:22:23.521527826Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:22:24.829625203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:22:24.833200801Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:22:26.762005788Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=warn ts=2026-02-16T06:22:28.5211957Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:22:30.564442287Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=debug ts=2026-02-16T06:22:30.868409055Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T06:22:32.314787341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:22:32.318413735Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:22:33.024921906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:22:33.028649563Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:22:33.028670103Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:22:33.028690955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:22:33.031824999Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:22:33.503594033Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:22:33.504646143Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:22:33.520949483Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:22:33.52097367Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:22:33.52098235Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:22:33.520982418Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:22:33.564159024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.7 kB" total_uncomp="55 kB" avg_uncomp="18 kB" full=3 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:22:33.567886044Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c65062932%3A19c65062958%3Af37ca9ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T06:22:33.567908364Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:22:33.567933362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:22:33.57148877Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T06:22:34.368157998Z caller=checkpoint.go:378 msg="writing series" size="89 kB"
level=info ts=2026-02-16T06:22:35.337470441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:22:35.341058075Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T06:22:35.341081641Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:22:35.341109859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:22:35.343980794Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T06:22:35.433243891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:22:35.43673877Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:22:35.436757332Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:22:35.64157322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:22:35.645311427Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T06:22:35.796336057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:22:38.170519133Z caller=checkpoint.go:378 msg="writing series" size="470 kB"
level=info ts=2026-02-16T06:22:38.275585784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:22:38.281943202Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:22:38.310240624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:22:38.313272628Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T06:22:38.521661066Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:22:41.972951977Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=warn ts=2026-02-16T06:22:43.521398107Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:22:45.394906039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:22:45.399756021Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:22:45.462848774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:22:45.466420385Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T06:22:45.776338061Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
level=info ts=2026-02-16T06:22:46.335968637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:22:46.339298278Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:22:46.339316693Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:22:46.339337335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:22:46.342166021Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T06:22:47.167927646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:22:47.171552472Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T06:22:47.793260077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="713 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:22:47.796830548Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T06:22:48.521419091Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:22:49.575869817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T06:22:49.578314308Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=error ts=2026-02-16T06:22:49.579454347Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:22:49.579472615Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:22:49.579492658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:22:49.582412719Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:22:50.147089628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:22:50.150682791Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:22:50.1507036Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:22:50.150727003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:22:50.153655362Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:22:50.88970751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:22:50.893052395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:22:50.89964764Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T06:22:50.899663059Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:22:50.899683366Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:22:50.899710073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:22:50.902807385Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:22:50.925035178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:22:50.92841368Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=debug ts=2026-02-16T06:22:52.147406334Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:22:52.658654467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:22:52.66175538Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:22:53.381527379Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=warn ts=2026-02-16T06:22:53.52147842Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:22:57.18422168Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=info ts=2026-02-16T06:22:57.261408095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:22:57.266718193Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T06:22:58.521634674Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:23:00.98721523Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=debug ts=2026-02-16T06:23:00.987238064Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=13
level=info ts=2026-02-16T06:23:01.419402603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="571 B" avg_comp="571 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:23:01.423260762Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:23:01.423278469Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:23:01.423301615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:23:01.426535991Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:23:01.818652524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:23:01.822200396Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:23:01.965568027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:23:01.969393738Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:23:01.969411257Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:23:01.969432771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:23:01.9723Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:23:02.906952879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:23:02.910644552Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:23:03.521527198Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:23:03.521555496Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:23:03.521565678Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:23:03.521562699Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:23:04.267474808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:23:04.273895635Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:23:04.722296704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:23:04.725577054Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:23:04.725596921Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:23:04.725617314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:23:04.729428153Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T06:23:04.791426189Z caller=checkpoint.go:378 msg="writing series" size="2.0 MB"
level=debug ts=2026-02-16T06:23:04.791443549Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=1
level=info ts=2026-02-16T06:23:05.209737127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:23:05.212793279Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T06:23:05.257789779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:23:05.260584574Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
ts=2026-02-16T06:23:05.798435066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:23:06.207094718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:57460"
level=info ts=2026-02-16T06:23:07.225876664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:23:07.229120901Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-02-16T06:23:08.099214297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:47262"
level=warn ts=2026-02-16T06:23:08.521285307Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:23:08.592555903Z caller=checkpoint.go:378 msg="writing series" size="228 kB"
level=info ts=2026-02-16T06:23:09.013353181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:23:09.016552601Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T06:23:09.051856598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T06:23:09.204400583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:23:09.207610137Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:23:10.084777191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="905 B" avg_comp="452 B" total_uncomp="689 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:23:10.087917719Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T06:23:12.396111867Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=info ts=2026-02-16T06:23:12.426218905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:23:12.429328137Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:23:13.188321599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:23:13.191762419Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T06:23:13.521494795Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:23:13.890983575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="12 kB" avg_comp="4.0 kB" total_uncomp="114 kB" avg_uncomp="38 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:23:13.894784973Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T06:23:16.198383231Z caller=checkpoint.go:378 msg="writing series" size="216 kB"
level=warn ts=2026-02-16T06:23:18.521684525Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:23:18.544819076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:23:18.549774602Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:23:20.001765331Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=info ts=2026-02-16T06:23:20.339625194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:23:20.343472078Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:23:20.601321836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:23:20.605572572Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T06:23:23.521782968Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:23:23.804913058Z caller=checkpoint.go:378 msg="writing series" size="647 kB"
level=debug ts=2026-02-16T06:23:23.804934634Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-02-16T06:23:24.087269871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:23:24.091270807Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T06:23:24.794265384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:23:24.79725422Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:23:26.269354842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:23:26.272354196Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:23:27.499960226Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T06:23:27.611107105Z caller=checkpoint.go:378 msg="writing series" size="4.2 MB"
level=debug ts=2026-02-16T06:23:27.611129168Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.2 MB" series=1
level=info ts=2026-02-16T06:23:27.643463093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:23:27.64701384Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T06:23:28.521033089Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:23:31.409353608Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=debug ts=2026-02-16T06:23:33.503410541Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:23:33.50447201Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:23:33.521157904Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:23:33.521177184Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:23:33.521185555Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:23:33.521189909Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:23:34.255904428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:23:34.259694941Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T06:23:34.793243397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:23:34.798092266Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T06:23:34.936479668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:23:34.943552522Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:23:35.212158091Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T06:23:35.401697849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:23:35.405610453Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T06:23:35.800221323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=info ts=2026-02-16T06:23:35.952513553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:23:35.955971514Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T06:23:38.521784287Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:23:39.015987921Z caller=checkpoint.go:378 msg="writing series" size="939 B"
level=info ts=2026-02-16T06:23:40.972346257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:23:40.975908103Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:23:42.514299732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:23:42.517659714Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T06:23:42.818505842Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=warn ts=2026-02-16T06:23:43.521186749Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:23:44.40440737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:23:44.407819918Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T06:23:46.620968454Z caller=checkpoint.go:378 msg="writing series" size="172 kB"
level=info ts=2026-02-16T06:23:48.241151122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:23:48.245127997Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=warn ts=2026-02-16T06:23:48.52089786Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:23:48.760741548Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:23:48.762456492Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T06:23:49.054919181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:23:49.05962804Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T06:23:50.450013628Z caller=checkpoint.go:378 msg="writing series" size="22 MB"
level=debug ts=2026-02-16T06:23:50.450040908Z caller=checkpoint.go:390 msg="flushing series" totalSize="22 MB" series=6
level=info ts=2026-02-16T06:23:51.206489887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:23:51.210139063Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:23:51.348545972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:23:51.351629221Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:23:52.686046742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:23:52.689267164Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T06:23:53.521775087Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:23:54.226789203Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-02-16T06:23:54.771417024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:23:54.775945504Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:23:58.029266979Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-02-16T06:23:58.520888716Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:23:58.602163854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:23:58.605952902Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T06:24:01.227025683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:44374"
level=info ts=2026-02-16T06:24:01.230344787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:24:01.234157472Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:24:01.234177029Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:24:01.234199404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:24:01.237056704Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T06:24:01.835798033Z caller=checkpoint.go:378 msg="writing series" size="5.9 MB"
level=debug ts=2026-02-16T06:24:01.835822383Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.0 MB" series=3
level=debug ts=2026-02-16T06:24:03.415960392Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:24:03.521611602Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:24:03.521639049Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:24:03.521651808Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:24:03.521653412Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:24:03.604978704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:24:03.608801453Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T06:24:05.635648627Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
ts=2026-02-16T06:24:05.801779975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=info ts=2026-02-16T06:24:06.075153265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="12 kB" avg_comp="4.0 kB" total_uncomp="114 kB" avg_uncomp="38 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:24:06.079004679Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:24:06.079025207Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:24:06.07905279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="12 kB" avg_comp="4.0 kB" total_uncomp="114 kB" avg_uncomp="38 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:24:06.081922874Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
ts=2026-02-16T06:24:06.210378517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:50038"
level=info ts=2026-02-16T06:24:06.343307932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:24:06.346628298Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:24:07.124696027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="905 B" avg_comp="452 B" total_uncomp="689 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:24:07.128723916Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T06:24:08.521085638Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:24:09.437985264Z caller=checkpoint.go:378 msg="writing series" size="357 kB"
level=info ts=2026-02-16T06:24:11.764150679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:24:11.769191139Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:24:12.626559185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:24:12.630128446Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:24:12.704378915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:24:12.708524616Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T06:24:13.240563276Z caller=checkpoint.go:378 msg="writing series" size="512 B"
level=warn ts=2026-02-16T06:24:13.52127857Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:24:15.716233086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:24:15.719568347Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:24:16.851845118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:24:16.855307917Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:24:17.044017724Z caller=checkpoint.go:378 msg="writing series" size="7.3 kB"
level=info ts=2026-02-16T06:24:17.606801251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="1.9 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:24:17.610276435Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T06:24:18.204008081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:24:18.20743221Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T06:24:18.246492711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:24:18.249918543Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T06:24:18.521443015Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:24:20.846329807Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T06:24:21.223407822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:24:21.228353675Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T06:24:23.52138199Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:24:24.63901748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="12 kB" avg_comp="4.0 kB" total_uncomp="114 kB" avg_uncomp="38 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:24:24.642835426Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T06:24:24.649157433Z caller=checkpoint.go:378 msg="writing series" size="222 kB"
level=info ts=2026-02-16T06:24:27.1923353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:24:27.19850424Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:24:28.289104623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:24:28.292977988Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=debug ts=2026-02-16T06:24:28.452570415Z caller=checkpoint.go:378 msg="writing series" size="220 kB"
level=warn ts=2026-02-16T06:24:28.521513126Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:24:31.979647245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:24:31.984210156Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T06:24:32.255094151Z caller=checkpoint.go:378 msg="writing series" size="274 kB"
level=debug ts=2026-02-16T06:24:32.255117831Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-02-16T06:24:33.503102302Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:24:33.50416051Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:24:33.521208765Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:24:33.521238272Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:24:33.521247058Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:24:33.521248116Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:24:34.519845201Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:24:34.519910519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="483 B" avg_comp="483 B" total_uncomp="40 kB" avg_uncomp="40 kB" idle=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:24:34.524023929Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
ts=2026-02-16T06:24:35.804106274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:24:36.057529919Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=info ts=2026-02-16T06:24:37.13747949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:24:37.14113933Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:24:37.942255424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:24:37.948492059Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T06:24:38.521798643Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:24:39.311910629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:24:39.315189376Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T06:24:39.86087165Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T06:24:40.354238256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:24:40.357805025Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:24:40.450973153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:24:40.454777895Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:24:40.587289898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="622 B" avg_comp="622 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:24:40.590723443Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:24:40.984919497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:24:40.988503304Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T06:24:43.521212519Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:24:43.663494955Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T06:24:46.372910725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:24:46.377309512Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:24:47.467081591Z caller=checkpoint.go:378 msg="writing series" size="535 kB"
level=warn ts=2026-02-16T06:24:48.521031624Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:24:48.535192572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="905 B" avg_comp="452 B" total_uncomp="689 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:24:48.538581991Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:24:48.58565484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="323 B" avg_comp="323 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:24:48.58877223Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:24:50.408185069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T06:24:50.477305983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:24:50.480958822Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:24:50.847076995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:24:50.851157747Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T06:24:51.269172203Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-02-16T06:24:52.66243183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:24:52.666880885Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:24:53.155447055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:24:53.159394083Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=warn ts=2026-02-16T06:24:53.521027617Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:24:55.073636941Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T06:24:55.073675491Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=6
level=info ts=2026-02-16T06:24:55.375820553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="12 kB" avg_comp="4.0 kB" total_uncomp="114 kB" avg_uncomp="38 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:24:55.379428704Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:24:57.845405886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:24:57.849022571Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:24:58.193906413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:24:58.19744036Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T06:24:58.521085381Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:24:58.875429677Z caller=checkpoint.go:378 msg="writing series" size="457 kB"
level=debug ts=2026-02-16T06:24:59.043517898Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T06:25:02.677654843Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T06:25:03.521774038Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:25:03.521807056Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:25:03.521818281Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:25:03.521821619Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:25:03.920566221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:25:03.924511374Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
ts=2026-02-16T06:25:05.805546415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=info ts=2026-02-16T06:25:06.075937952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:25:06.079667165Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T06:25:06.480797202Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T06:25:07.628956875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:25:07.633190115Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T06:25:08.521594004Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:25:10.283663704Z caller=checkpoint.go:378 msg="writing series" size="127 kB"
level=info ts=2026-02-16T06:25:12.854170989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:25:12.858165125Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T06:25:13.521742646Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:25:13.766661317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:25:13.770764861Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=debug ts=2026-02-16T06:25:14.086023402Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=info ts=2026-02-16T06:25:17.859208959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:25:17.863580858Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T06:25:17.888681398Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=warn ts=2026-02-16T06:25:18.521734307Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:25:21.691928169Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=info ts=2026-02-16T06:25:22.616173271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:25:22.620746635Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=warn ts=2026-02-16T06:25:23.521184526Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:25:23.609613835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:25:23.613767032Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T06:25:25.49403573Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=info ts=2026-02-16T06:25:25.623652693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:25:25.627286047Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:25:26.109800855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:25:26.117977619Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:25:27.21568824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:25:27.219152472Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:25:28.019422015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:25:28.022869477Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T06:25:28.521245367Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:25:28.709818531Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T06:25:29.298906594Z caller=checkpoint.go:378 msg="writing series" size="858 kB"
level=debug ts=2026-02-16T06:25:29.2989409Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=9
level=debug ts=2026-02-16T06:25:31.130124758Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:25:31.232551276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:47556"
level=info ts=2026-02-16T06:25:32.707479144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:25:32.713020773Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T06:25:33.100140635Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=info ts=2026-02-16T06:25:33.31698132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:25:33.320397774Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T06:25:33.502858926Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:25:33.503907849Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T06:25:33.521201779Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:25:33.521258576Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:25:33.521276234Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:25:33.521283438Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-02-16T06:25:35.807567775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:25:36.216351626Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:53064"
level=debug ts=2026-02-16T06:25:36.902894323Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
level=info ts=2026-02-16T06:25:37.643914853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:25:37.649434447Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:25:38.203151319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:25:38.206887001Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T06:25:38.521417124Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:25:39.198334149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="343 B" total_uncomp="1.0 kB" avg_uncomp="342 B" idle=1 full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:25:39.202066572Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:25:39.202086632Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:25:39.202108641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="343 B" total_uncomp="1.0 kB" avg_uncomp="342 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:25:39.205136326Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:25:39.730720048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="928 B" avg_comp="928 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:25:39.734588252Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T06:25:40.705207894Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=info ts=2026-02-16T06:25:40.941160039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:25:40.944700034Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:25:41.73064702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:25:41.733965544Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:25:42.191626752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:25:42.194917231Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:25:42.39324695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:25:42.396526898Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:25:42.590990683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="12 kB" avg_comp="4.0 kB" total_uncomp="114 kB" avg_uncomp="38 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:25:42.59443896Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T06:25:43.521299919Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:25:44.507924412Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=info ts=2026-02-16T06:25:47.036999557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:25:47.040636638Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:25:48.311139521Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=warn ts=2026-02-16T06:25:48.521514961Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:25:52.114497597Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB"
level=info ts=2026-02-16T06:25:52.117541528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="337 B" total_uncomp="1.0 kB" avg_uncomp="342 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:25:52.122142773Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T06:25:53.52132788Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:25:53.870039086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:25:53.875356058Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:25:53.875396764Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:25:53.875436606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:25:53.879030042Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:25:54.015958863Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:25:54.175978437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="671 kB" avg_uncomp="671 kB" idle=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:25:54.180343455Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=debug ts=2026-02-16T06:25:55.917460914Z caller=checkpoint.go:378 msg="writing series" size="718 kB"
level=info ts=2026-02-16T06:25:56.768149914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:25:56.772112499Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T06:25:58.521920406Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:25:59.71930632Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=info ts=2026-02-16T06:26:01.932250223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:26:01.936307917Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T06:26:02.518705704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:26:02.521999543Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:26:03.52177408Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T06:26:03.521780792Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:26:03.521807015Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:26:03.521819436Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:26:03.5228776Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.0 MB" series=8
level=info ts=2026-02-16T06:26:03.556324814Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-02-16T06:26:03.557461974Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.035543751s
ts=2026-02-16T06:26:05.80873775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=info ts=2026-02-16T06:26:05.910821838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:26:05.915080949Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:26:05.947127196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:26:05.950448314Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-02-16T06:26:06.21738316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:59984"
level=debug ts=2026-02-16T06:26:06.26152175Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T06:26:07.583318281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:26:07.587190703Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T06:26:08.521354296Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:26:08.567502289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:26:08.572403078Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:26:11.219647366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:26:11.223096389Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:26:12.228637868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:26:12.232708045Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=info ts=2026-02-16T06:26:12.697028808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="343 B" total_uncomp="1.0 kB" avg_uncomp="342 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:26:12.700333614Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T06:26:13.521791032Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:26:13.534881195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:26:13.541540009Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:26:13.541559004Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:26:13.54158301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:26:13.547899535Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:26:13.553868486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:26:13.55708201Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:26:15.82294712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:26:15.826203401Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T06:26:18.521115168Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:26:20.561590591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:26:20.565234357Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=warn ts=2026-02-16T06:26:23.520992923Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:26:24.202742857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:26:24.208355218Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:26:25.890893269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:26:25.895113626Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:26:25.993411311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:26:25.999220841Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:26:26.926622205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:26:26.929730032Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:26:27.675545371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:26:27.678814338Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T06:26:28.521955115Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:26:28.585057582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:26:28.588215913Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T06:26:29.211049624Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:26:29.236098901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:26:29.239515625Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:26:30.909349146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:26:30.915899349Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T06:26:31.236335479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:41616"
level=info ts=2026-02-16T06:26:31.950783256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="12 kB" avg_comp="4.0 kB" total_uncomp="114 kB" avg_uncomp="38 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:26:31.95444385Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T06:26:33.503297361Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:26:33.50438582Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:26:33.521533122Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T06:26:33.521543791Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:26:33.521563175Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:26:33.521571807Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:26:33.52159411Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:26:33.52170009Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=debug ts=2026-02-16T06:26:33.521948215Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=info ts=2026-02-16T06:26:33.644926153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:26:33.648532161Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:26:35.266479705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:26:35.278753338Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T06:26:35.810045119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=info ts=2026-02-16T06:26:35.833985755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:26:35.837742973Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:26:36.167573861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.4 kB" avg_comp="2.4 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:26:36.172596213Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
ts=2026-02-16T06:26:36.219307532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:49740"
level=info ts=2026-02-16T06:26:37.185810437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:26:37.191297326Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:26:37.272552887Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
ts=2026-02-16T06:26:38.110332851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:60352"
level=warn ts=2026-02-16T06:26:38.521589455Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:26:39.673497041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:26:39.677028687Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T06:26:41.022382983Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=debug ts=2026-02-16T06:26:41.112691163Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T06:26:42.1474796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:26:42.15107817Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T06:26:43.521605063Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:26:44.773076965Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T06:26:44.773100691Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=4
level=warn ts=2026-02-16T06:26:48.521034448Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:26:48.522142087Z caller=checkpoint.go:378 msg="writing series" size="123 kB"
level=info ts=2026-02-16T06:26:50.631027905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:26:50.635186563Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T06:26:51.525310777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:26:51.528960156Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:26:51.974065855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:26:51.980109851Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T06:26:52.27296453Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-02-16T06:26:53.521870052Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:26:56.022232152Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=info ts=2026-02-16T06:26:56.908280559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:26:56.916062226Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:26:57.197868081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:26:57.20231654Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T06:26:57.406074779Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:26:57.555421694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="343 B" total_uncomp="1.0 kB" avg_uncomp="342 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:26:57.558891032Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T06:26:58.521840618Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:26:59.426768009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:26:59.430933309Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=info ts=2026-02-16T06:26:59.632432225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:26:59.637563693Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T06:26:59.772141018Z caller=checkpoint.go:378 msg="writing series" size="2.5 kB"
ts=2026-02-16T06:27:01.238900275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:37980"
level=info ts=2026-02-16T06:27:02.643035805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:27:02.647265454Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:27:03.52132234Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:27:03.52135363Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:27:03.521378354Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:27:03.521399136Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:27:03.52249605Z caller=checkpoint.go:378 msg="writing series" size="158 kB"
ts=2026-02-16T06:27:05.812563583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=debug ts=2026-02-16T06:27:07.272165203Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-02-16T06:27:08.521463243Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:27:11.022799706Z caller=checkpoint.go:378 msg="writing series" size="409 kB"
level=info ts=2026-02-16T06:27:11.621124617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:27:11.62552636Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:27:12.670129021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:27:12.6760356Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T06:27:13.521146633Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:27:14.772330329Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=info ts=2026-02-16T06:27:16.440167228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:27:16.443624737Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:27:17.119722448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:27:17.123692788Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:27:17.346257885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:27:17.350099954Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:27:18.10104573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:27:18.104515815Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=warn ts=2026-02-16T06:27:18.521325016Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:27:18.522405258Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T06:27:20.59412546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:27:20.597471771Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:27:22.245998872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:27:22.249406909Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T06:27:22.272498162Z caller=checkpoint.go:378 msg="writing series" size="4.4 kB"
level=debug ts=2026-02-16T06:27:22.452000103Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:27:22.45358428Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T06:27:22.632223952Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:27:23.209657227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:27:23.212837532Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T06:27:23.521458026Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:27:23.844740715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:27:23.849385305Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T06:27:26.022234347Z caller=checkpoint.go:378 msg="writing series" size="94 kB"
level=debug ts=2026-02-16T06:27:26.022257538Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-02-16T06:27:26.808605812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:27:26.812053872Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:27:27.845266235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:27:27.850437303Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:27:27.898605705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="14 kB" avg_comp="4.6 kB" total_uncomp="114 kB" avg_uncomp="38 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:27:27.901726056Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:27:28.314875203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:27:28.318492851Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T06:27:28.521481492Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:27:29.772676389Z caller=checkpoint.go:378 msg="writing series" size="4.5 kB"
level=info ts=2026-02-16T06:27:32.606129169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:27:32.609754989Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:27:33.054273721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:27:33.058205933Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T06:27:33.502823307Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:27:33.503950843Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:27:33.520963573Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T06:27:33.520967285Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:27:33.520980868Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:27:33.520988082Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:27:33.5221661Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=info ts=2026-02-16T06:27:33.646412645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:27:33.649938023Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:27:34.87667482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:27:34.881308317Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
ts=2026-02-16T06:27:35.814223505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=info ts=2026-02-16T06:27:35.928218788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:27:35.931589274Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
ts=2026-02-16T06:27:36.222149133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:49098"
level=debug ts=2026-02-16T06:27:37.272863985Z caller=checkpoint.go:378 msg="writing series" size="470 kB"
level=info ts=2026-02-16T06:27:38.480084238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:27:38.486304255Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:27:38.486325141Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:27:38.486346573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:27:38.489684449Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T06:27:38.521637152Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:27:41.022980073Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=warn ts=2026-02-16T06:27:43.521063683Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:27:44.772268388Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
level=warn ts=2026-02-16T06:27:48.521891374Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:27:48.521912681Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T06:27:49.337044964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:27:49.348294155Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:27:50.348947227Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:27:51.562949064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="343 B" total_uncomp="1.0 kB" avg_uncomp="342 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:27:51.566720441Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:27:52.071085785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="571 B" avg_comp="571 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:27:52.074994562Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T06:27:52.272589702Z caller=checkpoint.go:378 msg="writing series" size="72 kB"
level=info ts=2026-02-16T06:27:52.575561254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:27:52.579379782Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T06:27:53.520884445Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:27:53.668215282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:27:53.673080604Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:27:54.617508632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:27:54.621058657Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T06:27:56.022859356Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=info ts=2026-02-16T06:27:57.55840334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:27:57.562276012Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:27:57.562297277Z caller=flush.go:240 component=ingester loop=8 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:27:57.562320826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:27:57.56527856Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=info ts=2026-02-16T06:27:58.174803473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:27:58.180111403Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=warn ts=2026-02-16T06:27:58.52177084Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:27:58.770005907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:27:58.774125766Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:27:59.577112032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:27:59.582219635Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T06:27:59.772906373Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=warn ts=2026-02-16T06:28:03.521240982Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:28:03.521234212Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:28:03.521264897Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:28:03.521275317Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:28:03.523668642Z caller=checkpoint.go:378 msg="writing series" size="2.3 MB"
level=debug ts=2026-02-16T06:28:03.523684939Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.3 MB" series=10
ts=2026-02-16T06:28:05.815738491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:28:06.223143972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:50950"
level=debug ts=2026-02-16T06:28:07.272529917Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=debug ts=2026-02-16T06:28:07.400828391Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:28:08.116032378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:44830"
level=warn ts=2026-02-16T06:28:08.52103247Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:28:08.608392149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="14 kB" avg_comp="4.6 kB" total_uncomp="114 kB" avg_uncomp="38 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:28:08.611627531Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:28:09.397837587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:28:09.402941637Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:28:09.402966108Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:28:09.402992993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:28:09.407378003Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:28:10.249508461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:28:10.25281709Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T06:28:10.252838424Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:28:10.252860319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:28:10.255479783Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:28:11.022630872Z caller=checkpoint.go:378 msg="writing series" size="8.2 kB"
level=info ts=2026-02-16T06:28:12.77671165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:28:12.780106259Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T06:28:12.844324656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:28:12.847322849Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T06:28:13.521043319Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:28:14.538837224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:28:14.542159554Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:28:14.573215391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:28:14.576585061Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=debug ts=2026-02-16T06:28:14.772293191Z caller=checkpoint.go:378 msg="writing series" size="276 kB"
level=info ts=2026-02-16T06:28:15.370809994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:28:15.373966864Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:28:15.943173289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:28:15.946708605Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=info ts=2026-02-16T06:28:17.11698602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:28:17.120130149Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T06:28:18.521716488Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:28:18.522917143Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=debug ts=2026-02-16T06:28:19.309523702Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:28:19.728275206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:28:19.732283647Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T06:28:20.306788419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:28:20.309997228Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:28:21.126278725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:28:21.129554327Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T06:28:21.620954136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="713 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:28:21.624084038Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T06:28:22.272894232Z caller=checkpoint.go:378 msg="writing series" size="477 kB"
level=debug ts=2026-02-16T06:28:22.272919206Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-02-16T06:28:22.820486008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:28:22.82515514Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:28:23.380577658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:28:23.386335974Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T06:28:23.386357809Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:28:23.386393525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:28:23.389020289Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:28:23.457393752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:28:23.460293884Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T06:28:23.521001901Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:28:25.658883407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:28:25.662268553Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:28:25.808799601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:28:25.81179468Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:28:25.811809565Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:28:25.811829987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:28:25.81434994Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T06:28:26.02650147Z caller=checkpoint.go:378 msg="writing series" size="7.0 MB"
level=debug ts=2026-02-16T06:28:26.02652436Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.0 MB" series=1
level=info ts=2026-02-16T06:28:27.531404755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:28:27.535103264Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T06:28:28.521205196Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:28:29.77198093Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
ts=2026-02-16T06:28:31.245497684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:39526"
level=info ts=2026-02-16T06:28:32.354476377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:28:32.360154945Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T06:28:33.502789055Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:28:33.503849819Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:28:33.520977334Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:28:33.52099671Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:28:33.521006815Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:28:33.52100913Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:28:33.522106085Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
ts=2026-02-16T06:28:35.817390809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:28:36.224389226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:35304"
level=info ts=2026-02-16T06:28:36.611830945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:28:36.61585176Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T06:28:37.272511689Z caller=checkpoint.go:378 msg="writing series" size="939 B"
ts=2026-02-16T06:28:38.118314439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:52340"
level=warn ts=2026-02-16T06:28:38.521880119Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:28:40.288552408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:28:40.293470883Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:28:40.763722691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:28:40.767303735Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T06:28:41.022188546Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=info ts=2026-02-16T06:28:43.090893737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:28:43.09733006Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T06:28:43.521160513Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:28:43.880500956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:28:43.884060628Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T06:28:44.772744032Z caller=checkpoint.go:378 msg="writing series" size="206 kB"
level=info ts=2026-02-16T06:28:45.327337331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:28:45.332515543Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:28:48.258495824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:28:48.262751374Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=warn ts=2026-02-16T06:28:48.521736879Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:28:48.542627304Z caller=checkpoint.go:378 msg="writing series" size="27 MB"
level=debug ts=2026-02-16T06:28:48.542647196Z caller=checkpoint.go:390 msg="flushing series" totalSize="28 MB" series=6
level=info ts=2026-02-16T06:28:49.536333345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:28:49.539868958Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:28:49.901859032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:28:49.907252675Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:28:50.022524692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="343 B" total_uncomp="1.0 kB" avg_uncomp="342 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:28:50.025915922Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:28:50.042948926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:28:50.045939942Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:28:50.898423002Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:28:52.098915121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:28:52.102526491Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T06:28:52.272926604Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=warn ts=2026-02-16T06:28:53.521131968Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:28:53.544200952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="14 kB" avg_comp="3.5 kB" total_uncomp="129 kB" avg_uncomp="32 kB" idle=1 full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:28:53.547487314Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T06:28:56.022571302Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T06:28:57.32689631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:28:57.330923356Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:28:57.330945383Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:28:57.33096868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:28:57.333839399Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T06:28:58.521828682Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:28:59.042991473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:28:59.046340549Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T06:28:59.778516335Z caller=checkpoint.go:378 msg="writing series" size="6.5 MB"
level=debug ts=2026-02-16T06:28:59.778539633Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.5 MB" series=3
level=info ts=2026-02-16T06:29:00.548324389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:29:00.551519573Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:29:03.035403375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:29:03.038535918Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:29:03.03855459Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:29:03.038574932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:29:03.041321222Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:29:03.521005939Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:29:03.521035284Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:29:03.521044973Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:29:03.521035818Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:29:03.52211922Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=debug ts=2026-02-16T06:29:05.424251304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T06:29:05.818968384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=debug ts=2026-02-16T06:29:07.272386231Z caller=checkpoint.go:378 msg="writing series" size="458 kB"
level=info ts=2026-02-16T06:29:07.979262093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:29:07.983087291Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:29:07.983112338Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:29:07.98314038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:29:07.986191514Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:29:08.307268007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:29:08.310590693Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:29:08.310608542Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:29:08.310628871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:29:08.313262907Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T06:29:08.521616975Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:29:09.439290657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:29:09.442788043Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:29:09.897060465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:29:09.900405725Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:29:10.35134867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:29:10.354634121Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6506243c%3A19c650697d1%3A3b570e67?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T06:29:10.354649617Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:29:10.354669756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="860 kB" avg_comp="860 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:29:10.358569843Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T06:29:11.021951447Z caller=checkpoint.go:378 msg="writing series" size="512 B"
level=info ts=2026-02-16T06:29:11.468006443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:29:11.471819895Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=warn ts=2026-02-16T06:29:13.521719276Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:29:14.772195849Z caller=checkpoint.go:378 msg="writing series" size="228 kB"
level=debug ts=2026-02-16T06:29:16.058778838Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:29:16.098938573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="323 B" avg_comp="323 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:29:16.102802043Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:29:16.102820185Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:29:16.102838775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:29:16.105787037Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:29:18.079045076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:29:18.082542732Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T06:29:18.521811818Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:29:18.522923976Z caller=checkpoint.go:378 msg="writing series" size="9.9 kB"
level=info ts=2026-02-16T06:29:19.241145063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:29:19.244643772Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:29:19.253707863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:29:19.256742082Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:29:19.346076855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.5 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T06:29:19.349440127Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c65062bf9%3A19c65062bfa%3A842bbbb2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T06:29:19.349456122Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:29:19.349478756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:29:19.353909475Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T06:29:20.165126674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:29:20.169538153Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T06:29:21.006974112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:29:21.010669859Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:29:22.262160864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:29:22.265479213Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:29:22.27253794Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-02-16T06:29:23.521732302Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:29:24.960959977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:29:24.964413209Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:29:24.964431707Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:29:24.964451628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:29:24.967068358Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:29:25.533778102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:29:25.537257229Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=debug ts=2026-02-16T06:29:26.022442267Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=info ts=2026-02-16T06:29:26.481539273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:29:26.484970228Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:29:27.794803177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:29:27.799229316Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T06:29:28.521831205Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:29:28.83477713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:29:28.83799771Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:29:28.838013876Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:29:28.838032172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:29:28.840838078Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T06:29:29.772406238Z caller=checkpoint.go:378 msg="writing series" size="334 kB"
level=debug ts=2026-02-16T06:29:29.77243168Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-02-16T06:29:30.371035982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:29:30.375201151Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T06:29:30.447401936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.5 kB" avg_comp="2.5 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:29:30.450466394Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:29:31.290685672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:29:31.294225617Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=info ts=2026-02-16T06:29:31.376777075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:29:31.382704977Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:29:33.503121885Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:29:33.50416269Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:29:33.5215265Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:29:33.521542916Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:29:33.521550277Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:29:33.521559864Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:29:33.522727977Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
ts=2026-02-16T06:29:35.820410797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=info ts=2026-02-16T06:29:36.964257015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:29:36.969238805Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T06:29:37.273029001Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=info ts=2026-02-16T06:29:37.314373363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:29:37.318211994Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:29:37.318229775Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:29:37.31824969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:29:37.321570764Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:29:37.488896377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="14 kB" avg_comp="3.5 kB" total_uncomp="129 kB" avg_uncomp="32 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:29:37.492627945Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T06:29:38.52135882Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:29:39.387104424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:29:39.392502121Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:29:40.214751519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:29:40.217793727Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:29:40.812356419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:29:40.815506743Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:29:40.815526059Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:29:40.815545326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:29:40.81825609Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T06:29:41.022882152Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T06:29:41.782721316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="343 B" total_uncomp="1.0 kB" avg_uncomp="342 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:29:41.786173484Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T06:29:42.407577957Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T06:29:43.521725391Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:29:44.772310212Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T06:29:46.919048278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:29:46.922529472Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:29:48.047497288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:29:48.051024058Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T06:29:48.521660764Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:29:48.523120561Z caller=checkpoint.go:378 msg="writing series" size="752 kB"
level=info ts=2026-02-16T06:29:50.102055023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:29:50.105710857Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:29:50.105729682Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:29:50.105749361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:29:50.108451732Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T06:29:51.595089551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:29:51.61452705Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:29:52.272373726Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-02-16T06:29:52.551792757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:29:52.555047816Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=info ts=2026-02-16T06:29:53.081347974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:29:53.084591377Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T06:29:53.52180573Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:29:54.07344307Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T06:29:54.577001172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:29:54.580611698Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:29:54.654801024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:29:54.657934513Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:29:54.882480485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="622 B" avg_comp="622 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:29:54.885778258Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T06:29:56.023541574Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T06:29:56.02356726Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.1 MB" series=7
level=warn ts=2026-02-16T06:29:58.521263802Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:29:58.647550347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:29:58.652025074Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T06:29:59.336495033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:29:59.339980885Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:29:59.506404067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:29:59.509691673Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:29:59.772714196Z caller=checkpoint.go:378 msg="writing series" size="520 kB"
level=info ts=2026-02-16T06:30:01.896176837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:30:01.900254274Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T06:30:02.663129126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:30:02.666286868Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:30:03.328286799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:30:03.331496935Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T06:30:03.520919731Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:30:03.520926161Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:30:03.520951882Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:30:03.520959975Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:30:03.522033865Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-02-16T06:30:05.821710469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=info ts=2026-02-16T06:30:06.022748965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:30:06.026219907Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:30:07.272511567Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T06:30:08.163116739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:30:08.167875553Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:30:08.167900571Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:30:08.167930762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:30:08.170990959Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T06:30:08.521719673Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:30:09.699044184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:30:09.702497898Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T06:30:11.022748714Z caller=checkpoint.go:378 msg="writing series" size="214 kB"
level=info ts=2026-02-16T06:30:11.819149815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:30:11.82299565Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T06:30:12.024197588Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T06:30:13.520895229Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:30:14.772262834Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=info ts=2026-02-16T06:30:14.92279525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:30:14.928136626Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:30:15.586931309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:30:15.590399515Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=warn ts=2026-02-16T06:30:18.521528185Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:30:18.522649646Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=info ts=2026-02-16T06:30:19.867683451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:30:19.871685756Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T06:30:21.438933882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="398 kB" avg_comp="398 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:30:21.443095845Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:30:21.89421726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:30:21.897667601Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:30:21.897687819Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:30:21.897710532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:30:21.900682914Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:30:22.242671089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:30:22.246531454Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T06:30:22.272665569Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=warn ts=2026-02-16T06:30:23.521322141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:30:23.643573003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="343 B" total_uncomp="1.0 kB" avg_uncomp="342 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:30:23.647041493Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T06:30:26.022334371Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=info ts=2026-02-16T06:30:27.405470843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:30:27.408934996Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:30:27.612474945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="301 kB" avg_uncomp="151 kB" idle=1 full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:30:27.615741647Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T06:30:28.521709437Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:30:29.015245254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:30:29.021280941Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:30:29.772735902Z caller=checkpoint.go:378 msg="writing series" size="919 kB"
level=debug ts=2026-02-16T06:30:29.772761237Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=9
level=info ts=2026-02-16T06:30:30.056975765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.8 kB" total_uncomp="96 kB" avg_uncomp="48 kB" idle=1 full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:30:30.06049849Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T06:30:32.490178673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:30:32.493750522Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:30:33.148519445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="14 kB" avg_comp="3.5 kB" total_uncomp="129 kB" avg_uncomp="32 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:30:33.152077256Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:30:33.152096769Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:30:33.1521191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="14 kB" avg_comp="3.5 kB" total_uncomp="129 kB" avg_uncomp="32 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:30:33.154950872Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T06:30:33.503109083Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:30:33.504171114Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T06:30:33.505488179Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-02-16T06:30:33.505527585Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1968025 n=1
level=debug ts=2026-02-16T06:30:33.505569693Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1968025 n=1
level=info ts=2026-02-16T06:30:33.520901154Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:30:33.52091928Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:30:33.52092701Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:30:33.520937367Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:30:33.522060355Z caller=checkpoint.go:378 msg="writing series" size="33 kB"
ts=2026-02-16T06:30:35.823194523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:30:36.232586606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:50542"
level=debug ts=2026-02-16T06:30:37.27261135Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
level=warn ts=2026-02-16T06:30:38.521419264Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:30:38.950596005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:30:38.957910355Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:30:39.368983144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:30:39.372602029Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:30:39.86197256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:30:39.865667704Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:30:40.69074777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="928 B" avg_comp="928 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:30:40.69414068Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T06:30:41.022900647Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=info ts=2026-02-16T06:30:41.293483883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:30:41.296780407Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=info ts=2026-02-16T06:30:43.041800636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:30:43.046452494Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T06:30:43.444663802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:30:43.44805233Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T06:30:43.521023729Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:30:44.772134956Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=debug ts=2026-02-16T06:30:47.33556453Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:30:47.455657317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:30:47.45955839Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T06:30:48.52142829Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:30:48.522494653Z caller=checkpoint.go:378 msg="writing series" size="2.4 kB"
level=info ts=2026-02-16T06:30:49.384653666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:30:49.387922178Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T06:30:49.484200653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:30:49.487413093Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:30:49.903123066Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T06:30:50.783476213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="14 kB" avg_comp="3.5 kB" total_uncomp="129 kB" avg_uncomp="32 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:30:50.786703273Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:30:51.522796249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:30:51.526095637Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:30:52.272155063Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=warn ts=2026-02-16T06:30:53.521000068Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:30:53.629378558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:30:53.634248356Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T06:30:56.022744666Z caller=checkpoint.go:378 msg="writing series" size="923 kB"
level=debug ts=2026-02-16T06:30:56.022771706Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=7
level=info ts=2026-02-16T06:30:58.516483097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:30:58.520007271Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T06:30:58.521035554Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:30:58.784131983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:30:58.787998038Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:30:59.772137871Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
ts=2026-02-16T06:31:01.2535866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:37882"
level=info ts=2026-02-16T06:31:03.521623286Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:31:03.521651443Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:31:03.521664644Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:31:03.521671975Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:31:03.522757369Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.8 kB" series=1
level=info ts=2026-02-16T06:31:03.543612501Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-02-16T06:31:03.544878745Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.023005592s
level=info ts=2026-02-16T06:31:04.013744163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="441 B" total_uncomp="1.0 kB" avg_uncomp="342 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:31:04.018892899Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:31:04.226941197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:31:04.231113079Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:31:05.025177774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:31:05.028521783Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:31:05.783503356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:31:05.787357362Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
ts=2026-02-16T06:31:05.825099418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=info ts=2026-02-16T06:31:05.848925139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="426 kB" avg_comp="213 kB" total_uncomp="4.3 MB" avg_uncomp="2.1 MB" idle=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:31:05.852339771Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
ts=2026-02-16T06:31:06.234208238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:51100"
level=info ts=2026-02-16T06:31:08.416063826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="311 kB" avg_uncomp="311 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:31:08.419327253Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T06:31:08.52134227Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:31:09.811293264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:31:09.815979256Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=info ts=2026-02-16T06:31:11.521270294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:31:11.525822699Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T06:31:13.521040513Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:31:16.530298416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="301 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:31:16.533987858Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:31:16.675184083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.8 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:31:16.678869996Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T06:31:16.802407833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="14 kB" avg_comp="3.5 kB" total_uncomp="129 kB" avg_uncomp="32 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:31:16.805766902Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T06:31:18.521284869Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:31:21.827284049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:31:21.832574858Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:31:22.731551645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:31:22.734988053Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T06:31:23.191145132Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T06:31:23.521860934Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:31:24.433375423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:31:24.436849382Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:31:25.603979873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:31:25.611213889Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T06:31:28.521063835Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:31:30.0577258Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:31:30.059617195Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T06:31:31.255748452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:56006"
level=info ts=2026-02-16T06:31:32.067488983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:31:32.070699559Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T06:31:33.503508947Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:31:33.50455069Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:31:33.52083548Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-02-16T06:31:33.520846954Z caller=checkpoint.go:612 msg="starting checkpoint"
level=debug ts=2026-02-16T06:31:33.520856673Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:31:33.520866303Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:31:33.52089584Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:31:33.520974787Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=debug ts=2026-02-16T06:31:33.521160608Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
ts=2026-02-16T06:31:35.826281932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=info ts=2026-02-16T06:31:36.839472195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:31:36.843326956Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:31:36.8433461Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:31:36.843387126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:31:36.846457191Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=debug ts=2026-02-16T06:31:37.271625761Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=info ts=2026-02-16T06:31:37.947099859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:31:37.950763594Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T06:31:38.521382555Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:31:39.232440501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:31:39.257927785Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T06:31:39.747643558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:31:39.751559278Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:31:39.981095498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:31:39.986697209Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:31:41.021710355Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=info ts=2026-02-16T06:31:42.15710943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:31:42.161737644Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T06:31:42.868342386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:31:42.872760118Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T06:31:43.521490526Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:31:44.772491064Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T06:31:44.772516103Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-02-16T06:31:46.916404775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:31:46.920089733Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T06:31:48.521660216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:31:48.521710713Z caller=checkpoint.go:378 msg="writing series" size="150 kB"
level=info ts=2026-02-16T06:31:48.757765586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:31:48.761443288Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:31:49.430297543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:31:49.433971082Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:31:50.281187975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:31:50.284817661Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=info ts=2026-02-16T06:31:50.373132904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:31:50.376854755Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T06:31:52.27115508Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T06:31:52.441812343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="447 B" total_uncomp="1.0 kB" avg_uncomp="342 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:31:52.445214614Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:31:52.445232608Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:31:52.445253077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="447 B" total_uncomp="1.0 kB" avg_uncomp="342 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:31:52.448052261Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T06:31:53.521807478Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:31:54.556252751Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T06:31:56.022320559Z caller=checkpoint.go:378 msg="writing series" size="328 kB"
level=info ts=2026-02-16T06:31:56.357091673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:31:56.36087562Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T06:31:58.521125861Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:31:59.771695891Z caller=checkpoint.go:378 msg="writing series" size="2.8 kB"
level=info ts=2026-02-16T06:32:00.828647193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="426 kB" avg_comp="213 kB" total_uncomp="4.3 MB" avg_uncomp="2.1 MB" full=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:32:00.832528658Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:32:01.557706056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:32:01.561411807Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:32:01.56142997Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:32:01.561449233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:32:01.564505679Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:32:03.521099313Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T06:32:03.521099367Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:32:03.521124268Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:32:03.521132481Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:32:03.521218716Z caller=checkpoint.go:378 msg="writing series" size="158 kB"
level=info ts=2026-02-16T06:32:03.631615957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:32:03.635601157Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:32:04.103727661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="301 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:32:04.107225043Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:32:04.442963014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="447 B" total_uncomp="1.0 kB" avg_uncomp="342 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:32:04.446176787Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:32:05.342759885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:32:05.346091728Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T06:32:05.827781951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=info ts=2026-02-16T06:32:06.383821961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="14 kB" avg_comp="3.5 kB" total_uncomp="129 kB" avg_uncomp="32 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:32:06.386947176Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T06:32:07.271325447Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=info ts=2026-02-16T06:32:07.677599496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:32:07.681458667Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=info ts=2026-02-16T06:32:07.951433241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="46 kB" avg_comp="23 kB" total_uncomp="321 kB" avg_uncomp="161 kB" idle=1 full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:32:07.959472216Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T06:32:08.521789848Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:32:08.694192938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:32:08.699521269Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T06:32:11.02193343Z caller=checkpoint.go:378 msg="writing series" size="613 kB"
level=debug ts=2026-02-16T06:32:11.021966093Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=7
level=info ts=2026-02-16T06:32:11.910623432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.8 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:32:11.914556942Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T06:32:13.521230657Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:32:14.771299174Z caller=checkpoint.go:378 msg="writing series" size="84 kB"
level=info ts=2026-02-16T06:32:15.37984234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:32:15.38379001Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T06:32:18.521885405Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:32:18.521907724Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T06:32:18.993667498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:32:18.997467688Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:32:20.585959169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:32:20.589529958Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=info ts=2026-02-16T06:32:21.521791784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:32:21.528211726Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:32:22.272048329Z caller=checkpoint.go:378 msg="writing series" size="4.4 kB"
level=warn ts=2026-02-16T06:32:23.521862001Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:32:24.20756013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:32:24.210871867Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T06:32:26.01405961Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T06:32:26.02221045Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
level=debug ts=2026-02-16T06:32:26.161331894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T06:32:26.37523837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="447 B" total_uncomp="1.0 kB" avg_uncomp="342 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:32:26.378401554Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:32:27.979875563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:32:27.982827811Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:32:27.982845438Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:32:27.982866108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:32:27.985506936Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T06:32:28.521096082Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:32:29.771515563Z caller=checkpoint.go:378 msg="writing series" size="4.5 kB"
level=debug ts=2026-02-16T06:32:33.503612206Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:32:33.504698269Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:32:33.521055963Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:32:33.521075652Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:32:33.521083689Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:32:33.521118779Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:32:33.521199087Z caller=checkpoint.go:378 msg="writing series" size="120 kB"
level=info ts=2026-02-16T06:32:34.287220222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:32:34.291008561Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:32:34.33634148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:32:34.339456596Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:32:34.371529568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:32:34.375548313Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T06:32:34.817804785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:32:34.821403829Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-02-16T06:32:35.829761104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:32:36.241132249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:37758"
level=debug ts=2026-02-16T06:32:37.272378488Z caller=checkpoint.go:378 msg="writing series" size="470 kB"
level=info ts=2026-02-16T06:32:37.382660444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:32:37.38616054Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T06:32:38.521867207Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:32:40.086157517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:32:40.090713232Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T06:32:40.515948968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:32:40.519389959Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:32:40.657613566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:32:40.660881099Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:32:41.010477124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:32:41.013914853Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:32:41.021995256Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=warn ts=2026-02-16T06:32:43.521863933Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:32:44.5033257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:32:44.508290498Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:32:44.771973181Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
level=info ts=2026-02-16T06:32:45.5382935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:32:45.542917905Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:32:47.954857805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:32:47.958724292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="426 kB" avg_comp="213 kB" total_uncomp="4.3 MB" avg_uncomp="2.1 MB" full=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:32:47.958771235Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:32:47.962089115Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T06:32:48.521850242Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-02-16T06:32:48.521863905Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:32:48.911976834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="301 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:32:48.915348088Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:32:52.27192514Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=debug ts=2026-02-16T06:32:52.819964928Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T06:32:53.521784315Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:32:53.744501734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:32:53.748422337Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=debug ts=2026-02-16T06:32:56.021734742Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=debug ts=2026-02-16T06:32:56.021760498Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
level=info ts=2026-02-16T06:32:56.808993406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.8 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:32:56.812624687Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T06:32:57.608963374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:32:57.612843593Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:32:57.792291726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:32:57.79768969Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:32:58.161623022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:32:58.165015366Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T06:32:58.521915223Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:32:59.771819277Z caller=checkpoint.go:378 msg="writing series" size="152 kB"
level=info ts=2026-02-16T06:33:02.065953144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="14 kB" avg_comp="3.5 kB" total_uncomp="129 kB" avg_uncomp="32 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:33:02.069689609Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:33:02.363519677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="46 kB" avg_comp="23 kB" total_uncomp="321 kB" avg_uncomp="161 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:33:02.367035906Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T06:33:03.521831458Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:33:03.521842476Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:33:03.521860067Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:33:03.521868314Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:33:03.52489872Z caller=checkpoint.go:378 msg="writing series" size="2.3 MB"
level=debug ts=2026-02-16T06:33:03.524911975Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.5 MB" series=2
level=info ts=2026-02-16T06:33:03.749790416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="713 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:33:03.753080357Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:33:04.041408926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:33:04.045188818Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-02-16T06:33:05.833568326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:33:06.24222203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:47916"
level=debug ts=2026-02-16T06:33:07.271606419Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
ts=2026-02-16T06:33:08.142264917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:48524"
level=info ts=2026-02-16T06:33:08.145649966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:33:08.149626037Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=warn ts=2026-02-16T06:33:08.521242284Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:33:09.573559174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:33:09.577289959Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T06:33:11.022141052Z caller=checkpoint.go:378 msg="writing series" size="9.0 kB"
level=warn ts=2026-02-16T06:33:13.521053811Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:33:14.771469442Z caller=checkpoint.go:378 msg="writing series" size="323 kB"
level=info ts=2026-02-16T06:33:16.263355604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:33:16.266818119Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T06:33:16.391308634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="447 B" total_uncomp="1.0 kB" avg_uncomp="342 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:33:16.395157035Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:33:17.993030382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:33:17.99976074Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:33:17.999784263Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:33:17.999809356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:33:18.005624698Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T06:33:18.521331149Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:33:18.521437105Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=debug ts=2026-02-16T06:33:20.525382572Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:33:20.801269198Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:33:21.287409714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:33:21.290562287Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T06:33:22.272187921Z caller=checkpoint.go:378 msg="writing series" size="477 kB"
level=debug ts=2026-02-16T06:33:22.272209266Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-02-16T06:33:23.344958263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:33:23.348215046Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T06:33:23.521852741Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:33:23.635121592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:33:23.638237589Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T06:33:26.031973121Z caller=checkpoint.go:378 msg="writing series" size="9.2 MB"
level=debug ts=2026-02-16T06:33:26.031996803Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.2 MB" series=1
level=info ts=2026-02-16T06:33:26.949707747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:33:26.953335161Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T06:33:28.521303878Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:33:29.772149973Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=info ts=2026-02-16T06:33:30.731976444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="571 B" avg_comp="571 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:33:30.735262947Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:33:30.855510066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:33:30.858764305Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:33:31.747230942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:33:31.751715303Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T06:33:33.260579307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:33:33.264079419Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T06:33:33.503020714Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:33:33.504055618Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:33:33.520809352Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:33:33.520828483Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:33:33.520836879Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:33:33.520859478Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:33:33.521927316Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T06:33:33.729225101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:33:33.733353129Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T06:33:33.90012434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:33:33.906618919Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:33:33.966701506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:33:33.970153879Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=info ts=2026-02-16T06:33:34.050882867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="426 kB" avg_comp="213 kB" total_uncomp="4.3 MB" avg_uncomp="2.1 MB" full=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:33:34.054490304Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
ts=2026-02-16T06:33:35.835816603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:33:36.243138471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:36124"
level=info ts=2026-02-16T06:33:36.305861656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:33:36.30918293Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=debug ts=2026-02-16T06:33:37.271495749Z caller=checkpoint.go:378 msg="writing series" size="939 B"
level=warn ts=2026-02-16T06:33:38.52158157Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:33:41.022238625Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=info ts=2026-02-16T06:33:43.227872788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="46 kB" avg_comp="23 kB" total_uncomp="321 kB" avg_uncomp="161 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:33:43.231329894Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T06:33:43.520948431Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:33:43.841757076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="301 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:33:43.845533586Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:33:44.747718602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:33:44.751595309Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T06:33:44.771821021Z caller=checkpoint.go:378 msg="writing series" size="228 kB"
level=info ts=2026-02-16T06:33:47.03120577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:33:47.037242502Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:33:47.711317446Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:33:48.221519117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:33:48.227415747Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:33:48.508566909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="14 kB" avg_comp="3.5 kB" total_uncomp="129 kB" avg_uncomp="32 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:33:48.512021354Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T06:33:48.521711403Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:33:48.546217093Z caller=checkpoint.go:378 msg="writing series" size="30 MB"
level=debug ts=2026-02-16T06:33:48.546239907Z caller=checkpoint.go:390 msg="flushing series" totalSize="31 MB" series=6
level=info ts=2026-02-16T06:33:51.746343987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:33:51.749947947Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=info ts=2026-02-16T06:33:51.894217277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.8 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:33:51.897843578Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T06:33:52.271806427Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-02-16T06:33:53.429203815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:33:53.43258817Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T06:33:53.520955443Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:33:56.021418914Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-02-16T06:33:58.521508256Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:33:59.58600059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:33:59.590001695Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:33:59.590019786Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:33:59.590044252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:33:59.593006628Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T06:33:59.77666567Z caller=checkpoint.go:378 msg="writing series" size="7.0 MB"
level=debug ts=2026-02-16T06:33:59.776690908Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.0 MB" series=3
level=info ts=2026-02-16T06:34:01.77705288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:34:01.783761367Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:34:02.188836267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:34:02.192383152Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:34:03.52176489Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:34:03.521763898Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=debug ts=2026-02-16T06:34:03.521792711Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T06:34:03.521797775Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:34:03.52180722Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-02-16T06:34:05.838782012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=info ts=2026-02-16T06:34:05.840282032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="447 B" total_uncomp="1.0 kB" avg_uncomp="342 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:34:05.843647464Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
ts=2026-02-16T06:34:06.244896547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:60194"
level=debug ts=2026-02-16T06:34:07.272218068Z caller=checkpoint.go:378 msg="writing series" size="349 kB"
level=warn ts=2026-02-16T06:34:08.521518591Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:34:08.963081982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:34:08.966422086Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:34:09.428486598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:34:09.432249782Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:34:10.249682414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:34:10.2530504Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:34:10.97111869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:34:10.974430735Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T06:34:11.021797752Z caller=checkpoint.go:378 msg="writing series" size="512 B"
level=debug ts=2026-02-16T06:34:12.44240745Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T06:34:13.520996443Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:34:14.771991365Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=info ts=2026-02-16T06:34:15.582009391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:34:15.585316508Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T06:34:15.6857007Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T06:34:18.521344703Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:34:18.52140129Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T06:34:19.209282781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:34:19.214418138Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:34:21.172500626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="323 B" avg_comp="323 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:34:21.176052218Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:34:22.271502623Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T06:34:23.49307337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:34:23.497504872Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-02-16T06:34:23.521356171Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:34:26.021804957Z caller=checkpoint.go:378 msg="writing series" size="147 kB"
level=info ts=2026-02-16T06:34:28.142253833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:34:28.147382805Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T06:34:28.485142086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:34:28.488921035Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:34:28.488941584Z caller=flush.go:240 component=ingester loop=8 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:34:28.488964743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:34:28.492058492Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=warn ts=2026-02-16T06:34:28.521014922Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:34:28.979072936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="14 kB" avg_comp="3.5 kB" total_uncomp="129 kB" avg_uncomp="32 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:34:28.982609553Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:34:29.448844683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="46 kB" avg_comp="23 kB" total_uncomp="321 kB" avg_uncomp="161 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:34:29.452317438Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T06:34:29.694952856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:34:29.698381024Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=debug ts=2026-02-16T06:34:29.771754257Z caller=checkpoint.go:378 msg="writing series" size="440 kB"
level=debug ts=2026-02-16T06:34:29.771779756Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-02-16T06:34:30.365486083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:34:30.368962883Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:34:31.242244674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:34:31.245477992Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T06:34:31.264889285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:42996"
level=info ts=2026-02-16T06:34:32.098136259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:34:32.103316672Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:34:32.103339441Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:34:32.103385437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:34:32.107585525Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:34:32.135670283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="301 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:34:32.138344948Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T06:34:32.138377241Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:34:32.274585752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="426 kB" avg_comp="213 kB" total_uncomp="4.3 MB" avg_uncomp="2.1 MB" full=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:34:32.278167218Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:34:32.403777581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="301 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:34:32.406789371Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:34:33.503547131Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:34:33.504606259Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:34:33.521023039Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:34:33.521047053Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:34:33.521058084Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:34:33.52105294Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:34:33.522170842Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=info ts=2026-02-16T06:34:34.276475327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:34:34.280027585Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T06:34:35.841798892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:34:37.271353678Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-02-16T06:34:38.067526367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:34:38.070827684Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=info ts=2026-02-16T06:34:38.144091494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:34:38.147285418Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T06:34:38.521333181Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:34:40.885488092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:34:40.888988207Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:34:41.022162234Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-02-16T06:34:41.265875946Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:34:42.542341757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:34:42.546139494Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=info ts=2026-02-16T06:34:43.094412275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:34:43.100799069Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T06:34:43.52159205Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:34:43.826105327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:34:43.829833607Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T06:34:44.771329682Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T06:34:45.512531893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.8 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:34:45.51622605Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T06:34:47.0339702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:34:47.039289782Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:34:47.916760972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="301 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:34:47.919904063Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T06:34:48.521702927Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:34:48.522023429Z caller=checkpoint.go:378 msg="writing series" size="761 kB"
level=info ts=2026-02-16T06:34:49.0089995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:34:49.01210251Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T06:34:50.008400812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="447 B" total_uncomp="1.0 kB" avg_uncomp="342 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:34:50.011497505Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T06:34:52.271709526Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-02-16T06:34:53.191670949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:34:53.195459815Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T06:34:53.521482643Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:34:56.023923691Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-02-16T06:34:56.023948717Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=7
level=info ts=2026-02-16T06:34:57.569791284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:34:57.573609896Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T06:34:58.521678205Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:34:59.772313893Z caller=checkpoint.go:378 msg="writing series" size="467 kB"
level=info ts=2026-02-16T06:35:03.520973764Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:35:03.521005222Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:35:03.521013747Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:35:03.521018844Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:35:03.522095038Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T06:35:04.081170999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:35:04.085594819Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
ts=2026-02-16T06:35:05.843843338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:35:06.248641501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:41428"
level=debug ts=2026-02-16T06:35:07.272140219Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T06:35:07.526972281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="622 B" avg_comp="622 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:35:07.530338723Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T06:35:07.530357753Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:35:07.530398444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:35:07.533115622Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-02-16T06:35:08.149713868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:57668"
level=info ts=2026-02-16T06:35:08.196395574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:35:08.201529375Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T06:35:08.513517948Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T06:35:08.52157681Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:35:09.523137676Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:35:10.795541404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="301 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:35:10.798961908Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:35:11.021572049Z caller=checkpoint.go:378 msg="writing series" size="212 kB"
level=info ts=2026-02-16T06:35:13.027413845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="48 kB" avg_comp="24 kB" total_uncomp="321 kB" avg_uncomp="161 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:35:13.030770292Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:35:13.030792556Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:35:13.030814339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="48 kB" avg_comp="24 kB" total_uncomp="321 kB" avg_uncomp="161 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:35:13.033594399Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T06:35:13.521714818Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:35:13.773678576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:35:13.777170779Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T06:35:14.772170511Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=info ts=2026-02-16T06:35:15.024869357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:35:15.028148361Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:35:15.745294094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:35:15.748466733Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:35:17.511057243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:35:17.514440311Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:35:18.094675936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:35:18.097745798Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:35:18.521320738Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-02-16T06:35:18.521349076Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:35:19.623623807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:35:19.6277031Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:35:19.778318259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:35:19.781742748Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=info ts=2026-02-16T06:35:21.480086587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:35:21.484849294Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T06:35:22.272105026Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=info ts=2026-02-16T06:35:22.398558788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:35:22.403280401Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-02-16T06:35:23.520867731Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:35:24.337403816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="48 kB" avg_comp="24 kB" total_uncomp="321 kB" avg_uncomp="161 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:35:24.340695235Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T06:35:25.239018824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="14 kB" avg_comp="3.5 kB" total_uncomp="129 kB" avg_uncomp="32 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:35:25.242230539Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T06:35:26.021548427Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=info ts=2026-02-16T06:35:26.813765733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:35:26.81963966Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:35:27.120769368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="928 B" avg_comp="928 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:35:27.124336249Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:35:27.124354659Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:35:27.124398194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:35:27.127480709Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:35:27.414276237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.8 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:35:27.417843607Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T06:35:28.52116499Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:35:29.773815714Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T06:35:29.773839845Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=9
ts=2026-02-16T06:35:31.26831269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:49498"
level=info ts=2026-02-16T06:35:31.44840927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="426 kB" avg_comp="213 kB" total_uncomp="4.3 MB" avg_uncomp="2.1 MB" full=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:35:31.452814427Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:35:31.469870272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:35:31.472758593Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:35:33.295720484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="344 B" idle=1 full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:35:33.29928421Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T06:35:33.503076212Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:35:33.504126091Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:35:33.521751335Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:35:33.521775895Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:35:33.521784789Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:35:33.521804256Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:35:33.521841522Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=info ts=2026-02-16T06:35:35.808473393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:35:35.812075874Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-02-16T06:35:35.845656557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:35:37.272253498Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=warn ts=2026-02-16T06:35:38.520979969Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:35:40.375552916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:35:40.379469504Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T06:35:41.022349794Z caller=checkpoint.go:378 msg="writing series" size="88 kB"
level=info ts=2026-02-16T06:35:41.567833837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:35:41.57523711Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:35:43.134677223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:35:43.13814634Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:35:43.138164476Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:35:43.138185151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:35:43.140961373Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T06:35:43.521657824Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:35:43.67100871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:35:43.674375263Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T06:35:44.77173759Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=debug ts=2026-02-16T06:35:45.060518976Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:35:47.036207348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:35:47.039762018Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=info ts=2026-02-16T06:35:47.714645576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:35:47.717777514Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T06:35:47.964982229Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:35:47.966462272Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T06:35:48.521303115Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:35:48.521313039Z caller=checkpoint.go:378 msg="writing series" size="2.4 kB"
level=info ts=2026-02-16T06:35:49.966479025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:35:49.969745836Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:35:49.969765883Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:35:49.969788124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:35:49.972540581Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:35:51.225089475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="48 kB" avg_comp="24 kB" total_uncomp="321 kB" avg_uncomp="161 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:35:51.228680241Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T06:35:52.271710652Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=info ts=2026-02-16T06:35:53.084107817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:35:53.090340978Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:35:53.09037524Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:35:53.090396927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:35:53.094189103Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T06:35:53.521309933Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:35:55.404739114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:35:55.408058402Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T06:35:56.022066349Z caller=checkpoint.go:378 msg="writing series" size="982 kB"
level=debug ts=2026-02-16T06:35:56.022090345Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=7
level=info ts=2026-02-16T06:35:56.770546891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:35:56.774422694Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:35:58.31686494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:35:58.320243057Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:35:58.320259652Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:35:58.32027866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:35:58.323287352Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T06:35:58.521399955Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:35:59.493855358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="301 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:35:59.497221729Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:35:59.520466005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:35:59.523376743Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:35:59.740979393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:35:59.744346309Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:35:59.744393339Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:35:59.744413857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:35:59.747235892Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T06:35:59.771332237Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
ts=2026-02-16T06:36:01.270337412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:58626"
level=debug ts=2026-02-16T06:36:03.521300094Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.8 kB" series=1
level=info ts=2026-02-16T06:36:03.521314779Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:36:03.521335871Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:36:03.52134326Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:36:03.521348353Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:36:03.527950603Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-02-16T06:36:03.529009732Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007915993s
ts=2026-02-16T06:36:05.847634804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
ts=2026-02-16T06:36:08.153815176Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:36904"
level=warn ts=2026-02-16T06:36:08.521801914Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:36:08.977042133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:36:08.98066009Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:36:09.098987119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.8 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:36:09.102338233Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c65062956%3A19c650eb9da%3A75e9af9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T06:36:09.102356504Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:36:09.102397506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="282 kB" avg_uncomp="282 kB" idle=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:36:09.105255024Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=info ts=2026-02-16T06:36:09.228524932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:36:09.232596879Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:36:09.232613135Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:36:09.403061623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:36:09.407270088Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T06:36:11.242060031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:36:11.245575863Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:36:11.245599436Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:36:11.245625765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:36:11.248471898Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:36:11.262548062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:36:11.265412742Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:36:11.496212921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:36:11.499815015Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=info ts=2026-02-16T06:36:12.101097306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:36:12.105609201Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T06:36:12.404349042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:36:12.408013385Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:36:12.802956308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="14 kB" avg_comp="3.5 kB" total_uncomp="129 kB" avg_uncomp="32 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:36:12.806489433Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:36:12.956967959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:36:12.960160411Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:36:12.960178216Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:36:12.960199203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:36:12.962855068Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T06:36:13.520958318Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:36:13.626748466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:36:13.630049053Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:36:15.020310472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="362 B" total_uncomp="1.4 kB" avg_uncomp="344 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:36:15.023731038Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:36:15.109111342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:36:15.113964123Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:36:17.740106596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:36:17.743421778Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T06:36:18.520998185Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:36:18.970201278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:36:18.97340867Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:36:18.973427756Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:36:18.973449849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:36:18.976094972Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T06:36:20.527059168Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:36:20.547087646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:36:20.550348156Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:36:22.910620207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:36:22.913946253Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T06:36:23.521212173Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:36:24.501905794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:36:24.505868328Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T06:36:25.508766904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T06:36:25.565840337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:36:25.568722687Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:36:27.147933035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="282 kB" avg_uncomp="282 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:36:27.151507518Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=warn ts=2026-02-16T06:36:28.521714537Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:36:30.323791509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:36:30.327331308Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:36:31.066631155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:36:31.069952177Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:36:31.318731103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:36:31.325005153Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:36:31.379069778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="426 kB" avg_comp="213 kB" total_uncomp="4.3 MB" avg_uncomp="2.1 MB" full=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:36:31.382480028Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:36:31.382493337Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:36:31.382513349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="426 kB" avg_comp="213 kB" total_uncomp="4.3 MB" avg_uncomp="2.1 MB" full=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:36:31.385617591Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T06:36:33.503088594Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:36:33.504154972Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:36:33.516418141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="48 kB" avg_comp="24 kB" total_uncomp="321 kB" avg_uncomp="161 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:36:33.519497254Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T06:36:33.52156396Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:36:33.521561296Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T06:36:33.521582078Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:36:33.52159843Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:36:33.521606807Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T06:36:33.521683087Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=debug ts=2026-02-16T06:36:33.521849013Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=info ts=2026-02-16T06:36:34.396086615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:36:34.39928984Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:36:34.443334037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:36:34.446246878Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:36:34.446260204Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:36:34.446279476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:36:34.448827308Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:36:34.739849314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:36:34.743348813Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=info ts=2026-02-16T06:36:35.809407906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:36:35.813730873Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-02-16T06:36:35.84930247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:36:36.253673938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:55902"
level=debug ts=2026-02-16T06:36:37.272053064Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=warn ts=2026-02-16T06:36:38.521377821Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:36:40.162340067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:36:40.16580903Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=debug ts=2026-02-16T06:36:41.022026363Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=info ts=2026-02-16T06:36:42.744193688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:36:42.747563166Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T06:36:43.521543871Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:36:44.772822317Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-02-16T06:36:44.772848896Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=4
level=debug ts=2026-02-16T06:36:46.302282599Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:36:46.929956581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:36:46.933432329Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T06:36:47.628089516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="426 kB" avg_comp="213 kB" total_uncomp="4.3 MB" avg_uncomp="2.1 MB" full=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:36:47.631836003Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:36:48.025973643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:36:48.029968017Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T06:36:48.521643685Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:36:48.522781559Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
level=info ts=2026-02-16T06:36:51.041810408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:36:51.045607329Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:36:51.4493265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:36:51.453338671Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T06:36:52.272418239Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T06:36:52.995087382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:36:52.998491887Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T06:36:53.521690569Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:36:54.370278878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="16 kB" avg_comp="3.9 kB" total_uncomp="129 kB" avg_uncomp="32 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:36:54.375007884Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:36:54.375027173Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:36:54.375048679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="16 kB" avg_comp="3.9 kB" total_uncomp="129 kB" avg_uncomp="32 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:36:54.377718382Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:36:54.960386454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="282 kB" avg_uncomp="282 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:36:54.96364833Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=info ts=2026-02-16T06:36:55.362766333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="344 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:36:55.365784357Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T06:36:56.022760642Z caller=checkpoint.go:378 msg="writing series" size="228 kB"
level=info ts=2026-02-16T06:36:57.284252239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:36:57.287437357Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T06:36:58.521090066Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:36:58.759867665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="301 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:36:58.763010884Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:36:59.772860968Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=info ts=2026-02-16T06:37:02.378799661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:37:02.383683639Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:37:03.072243854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:37:03.075720292Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:37:03.521546165Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:37:03.521580119Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:37:03.52159071Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:37:03.521595861Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:37:03.522685078Z caller=checkpoint.go:378 msg="writing series" size="158 kB"
level=info ts=2026-02-16T06:37:04.270074672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:37:04.273845845Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:37:04.769261282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:37:04.772646142Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T06:37:05.850769681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:37:06.255414172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:33026"
level=info ts=2026-02-16T06:37:06.86557772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="16 kB" avg_comp="3.9 kB" total_uncomp="129 kB" avg_uncomp="32 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:37:06.868892538Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:37:06.936245656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:37:06.939634244Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T06:37:07.272420742Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
ts=2026-02-16T06:37:08.157076427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:56048"
level=info ts=2026-02-16T06:37:08.302945599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="426 kB" avg_comp="213 kB" total_uncomp="4.3 MB" avg_uncomp="2.1 MB" full=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:37:08.306638291Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:37:08.320633454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:37:08.323428851Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=warn ts=2026-02-16T06:37:08.521587447Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:37:11.022453499Z caller=checkpoint.go:378 msg="writing series" size="538 kB"
level=debug ts=2026-02-16T06:37:11.022478082Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-02-16T06:37:11.841003211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:37:11.844647022Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:37:11.844666197Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:37:11.844691074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:37:11.847462377Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:37:12.935890774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:37:12.939526889Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T06:37:13.520907032Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:37:14.772585524Z caller=checkpoint.go:378 msg="writing series" size="92 kB"
level=info ts=2026-02-16T06:37:16.891049152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:37:16.894782904Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=debug ts=2026-02-16T06:37:16.91707468Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T06:37:18.521159721Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:37:18.522243009Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T06:37:21.348615507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="48 kB" avg_comp="16 kB" total_uncomp="322 kB" avg_uncomp="107 kB" idle=1 full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:37:21.351775982Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T06:37:22.152757298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:37:22.272093685Z caller=checkpoint.go:378 msg="writing series" size="4.4 kB"
level=warn ts=2026-02-16T06:37:23.521036676Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:37:24.213702509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:37:24.216820098Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T06:37:26.022727011Z caller=checkpoint.go:378 msg="writing series" size="125 kB"
level=info ts=2026-02-16T06:37:26.855877209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:37:26.86361178Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:37:27.514152465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:37:27.517453023Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T06:37:28.521513811Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:37:29.245406238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:37:29.24864701Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:37:29.296814448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="16 kB" avg_comp="3.9 kB" total_uncomp="129 kB" avg_uncomp="32 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:37:29.299612393Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:37:29.393932467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:37:29.396786583Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T06:37:29.771911151Z caller=checkpoint.go:378 msg="writing series" size="4.5 kB"
level=info ts=2026-02-16T06:37:32.180409731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:37:32.184789852Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T06:37:33.502751428Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:37:33.503814625Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:37:33.52167286Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T06:37:33.521674141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:37:33.521692271Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:37:33.52169998Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:37:33.522871327Z caller=checkpoint.go:378 msg="writing series" size="133 kB"
ts=2026-02-16T06:37:35.852485535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=info ts=2026-02-16T06:37:36.238153297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:37:36.241672193Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:37:36.934626358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:37:36.937959779Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:37:37.272875051Z caller=checkpoint.go:378 msg="writing series" size="470 kB"
ts=2026-02-16T06:37:38.159021404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:48932"
level=warn ts=2026-02-16T06:37:38.521076165Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:37:38.699707636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:37:38.702954449Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T06:37:41.022658261Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T06:37:41.95716227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="282 kB" avg_uncomp="282 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:37:41.961239784Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=info ts=2026-02-16T06:37:42.452406236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:37:42.455743814Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T06:37:43.52139446Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:37:44.772711897Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
level=info ts=2026-02-16T06:37:44.964274636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:37:44.967758901Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:37:45.905191804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="301 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:37:45.908313327Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T06:37:48.521761469Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:37:48.522819474Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T06:37:50.200739849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:37:50.204337623Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:37:50.227313244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="344 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:37:50.230425389Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:37:50.230439164Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:37:50.230459105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="344 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:37:50.233164899Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:37:50.93776557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:37:50.9419391Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T06:37:51.453519261Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:37:51.861493698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:37:51.864896636Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:37:51.99028797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="713 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:37:51.993455921Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T06:37:52.272248702Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=warn ts=2026-02-16T06:37:53.521670664Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:37:53.564772071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:37:53.567922539Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T06:37:56.022496915Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=debug ts=2026-02-16T06:37:56.022524502Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=12
level=debug ts=2026-02-16T06:37:56.371754495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T06:37:58.521307292Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:37:59.772469072Z caller=checkpoint.go:378 msg="writing series" size="199 kB"
level=info ts=2026-02-16T06:38:00.168474593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="426 kB" avg_comp="213 kB" total_uncomp="4.3 MB" avg_uncomp="2.1 MB" full=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:38:00.172419559Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:38:01.037828742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:38:01.04124372Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:38:02.151236529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:38:02.156503555Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:38:02.366092068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:38:02.369650269Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:38:02.814620942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:38:02.817952523Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:38:02.817974177Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:38:02.817995781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:38:02.820846682Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=info ts=2026-02-16T06:38:03.521528962Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:38:03.521552834Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:38:03.521561031Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:38:03.521558729Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:38:03.524831873Z caller=checkpoint.go:378 msg="writing series" size="2.4 MB"
level=debug ts=2026-02-16T06:38:03.52484564Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.6 MB" series=2
ts=2026-02-16T06:38:05.854572872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=info ts=2026-02-16T06:38:05.971767724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="344 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:38:05.975288591Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T06:38:07.272792189Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=info ts=2026-02-16T06:38:07.322045405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="48 kB" avg_comp="16 kB" total_uncomp="322 kB" avg_uncomp="107 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:38:07.325641359Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T06:38:08.521171427Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:38:09.916213554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:38:09.920579349Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=debug ts=2026-02-16T06:38:11.022513259Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-02-16T06:38:12.187568966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="16 kB" avg_comp="3.9 kB" total_uncomp="129 kB" avg_uncomp="32 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:38:12.191122484Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T06:38:13.520870435Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:38:14.01493198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:38:14.018570446Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T06:38:14.772566171Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
level=info ts=2026-02-16T06:38:15.798419755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:38:15.805671654Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:38:16.799795785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:38:16.803070326Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=info ts=2026-02-16T06:38:17.391528475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:38:17.39492684Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:38:17.394946323Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:38:17.394968868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:38:17.397746822Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=warn ts=2026-02-16T06:38:18.521563548Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:38:18.522753451Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=info ts=2026-02-16T06:38:18.81448972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:38:18.818766044Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T06:38:19.584942212Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:38:19.740411885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:38:19.743920252Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:38:20.26227392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:38:20.265468431Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:38:21.557177818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:38:21.56032689Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T06:38:22.273087931Z caller=checkpoint.go:378 msg="writing series" size="426 kB"
level=warn ts=2026-02-16T06:38:23.521129121Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:38:24.76707951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="282 kB" avg_uncomp="282 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:38:24.770780261Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=debug ts=2026-02-16T06:38:26.032513409Z caller=checkpoint.go:378 msg="writing series" size="9.2 MB"
level=debug ts=2026-02-16T06:38:26.032537774Z caller=checkpoint.go:390 msg="flushing series" totalSize="10 MB" series=6
level=info ts=2026-02-16T06:38:27.593407114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:38:27.596637886Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T06:38:28.521446296Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:38:29.77212261Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=info ts=2026-02-16T06:38:32.421953902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:38:32.427337573Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:38:32.940009046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:38:32.943773852Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:38:33.503381913Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:38:33.504398749Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:38:33.521533427Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:38:33.521552609Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:38:33.521561534Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:38:33.521566002Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:38:33.522656817Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T06:38:35.254181094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="301 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:38:35.257831768Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
ts=2026-02-16T06:38:35.855572275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=info ts=2026-02-16T06:38:36.21712269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="571 B" avg_comp="571 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:38:36.220523049Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T06:38:36.261322518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:42388"
level=debug ts=2026-02-16T06:38:37.272862917Z caller=checkpoint.go:378 msg="writing series" size="939 B"
level=info ts=2026-02-16T06:38:38.078023216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:38:38.081677071Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T06:38:38.521098296Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:38:38.862898295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:38:38.866338732Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:38:40.795393747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="426 kB" avg_comp="142 kB" total_uncomp="4.3 MB" avg_uncomp="1.4 MB" idle=1 full=1 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:38:40.799262852Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T06:38:41.022117479Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=info ts=2026-02-16T06:38:41.289654913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="344 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:38:41.293001797Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:38:41.889579793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:38:41.892851547Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T06:38:43.52122611Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:38:44.319619974Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T06:38:44.772759379Z caller=checkpoint.go:378 msg="writing series" size="239 kB"
level=info ts=2026-02-16T06:38:46.084007238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:38:46.088495471Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:38:48.138005082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:38:48.14257642Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-02-16T06:38:48.521576149Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:38:48.54227572Z caller=checkpoint.go:378 msg="writing series" size="31 MB"
level=debug ts=2026-02-16T06:38:48.542299984Z caller=checkpoint.go:390 msg="flushing series" totalSize="32 MB" series=6
level=info ts=2026-02-16T06:38:50.094106908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:38:50.097461897Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:38:50.097484942Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:38:50.097512951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:38:50.100323065Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:38:51.413050372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:38:51.416543347Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=info ts=2026-02-16T06:38:51.931571783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:38:51.935090043Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=debug ts=2026-02-16T06:38:52.271908509Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=debug ts=2026-02-16T06:38:52.501346123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T06:38:53.521017095Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:38:56.022582646Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T06:38:56.44798513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:38:56.453552423Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:38:56.651322524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="48 kB" avg_comp="16 kB" total_uncomp="322 kB" avg_uncomp="107 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:38:56.654681831Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T06:38:58.521282733Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:38:59.776916039Z caller=checkpoint.go:378 msg="writing series" size="7.9 MB"
level=debug ts=2026-02-16T06:38:59.776942158Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.9 MB" series=3
level=info ts=2026-02-16T06:39:00.329922774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:39:00.333701462Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:39:00.950270547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:39:00.958203728Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:39:03.327339514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:39:03.333437793Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T06:39:03.521193569Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:39:03.521230359Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:39:03.521238956Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:39:03.521257056Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:39:03.522380472Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
ts=2026-02-16T06:39:05.856985075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=info ts=2026-02-16T06:39:06.245957187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=5 total_comp="16 kB" avg_comp="3.2 kB" total_uncomp="143 kB" avg_uncomp="29 kB" idle=1 full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:39:06.249824297Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:39:06.806396143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="282 kB" avg_uncomp="282 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:39:06.809893424Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=debug ts=2026-02-16T06:39:07.272225544Z caller=checkpoint.go:378 msg="writing series" size="469 kB"
level=warn ts=2026-02-16T06:39:08.52087749Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:39:09.179313766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:39:09.183163301Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:39:09.709763019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:39:09.713196781Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:39:10.068026884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:39:10.071746305Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:39:10.337659818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:39:10.342147821Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T06:39:11.021900684Z caller=checkpoint.go:378 msg="writing series" size="512 B"
level=warn ts=2026-02-16T06:39:13.521280622Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:39:14.097402579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:39:14.101099114Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:39:14.746284914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:39:14.75004806Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T06:39:14.772212241Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=warn ts=2026-02-16T06:39:18.521616187Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:39:18.522733371Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=debug ts=2026-02-16T06:39:18.901769853Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:39:19.814539024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:39:19.818161169Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T06:39:22.272010793Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T06:39:23.446553964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="622 B" avg_comp="622 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:39:23.449759321Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T06:39:23.521475889Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:39:23.800550995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="344 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:39:23.804077273Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:39:25.775911084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:39:25.779567652Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:39:26.022507047Z caller=checkpoint.go:378 msg="writing series" size="194 kB"
level=info ts=2026-02-16T06:39:26.41565353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:39:26.419114939Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:39:26.662049993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:39:26.665243675Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T06:39:28.521747505Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:39:29.772603869Z caller=checkpoint.go:378 msg="writing series" size="377 kB"
level=debug ts=2026-02-16T06:39:29.772626269Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-02-16T06:39:32.345763848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="301 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:39:32.349590856Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:39:32.878827283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:39:32.882765733Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T06:39:33.457350237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:39:33.462085875Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:39:33.503751425Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T06:39:33.503779632Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:39:33.521215808Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:39:33.52123411Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:39:33.521241686Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:39:33.52125928Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:39:33.522389067Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=info ts=2026-02-16T06:39:35.537928321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:39:35.541031895Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T06:39:35.859399441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=info ts=2026-02-16T06:39:37.181297281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="426 kB" avg_comp="142 kB" total_uncomp="4.3 MB" avg_uncomp="1.4 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:39:37.184877945Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T06:39:37.272156755Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
ts=2026-02-16T06:39:38.167789337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:52168"
level=warn ts=2026-02-16T06:39:38.521746198Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:39:39.483707079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:39:39.487664853Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=debug ts=2026-02-16T06:39:41.022729571Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-02-16T06:39:42.046610094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:39:42.048082706Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T06:39:42.481139958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:39:42.484380486Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:39:42.918606801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:39:42.921950973Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=warn ts=2026-02-16T06:39:43.521372298Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:39:44.772491925Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T06:39:45.923767429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:39:45.927416561Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T06:39:47.665765927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=5 total_comp="16 kB" avg_comp="3.2 kB" total_uncomp="143 kB" avg_uncomp="29 kB" full=5 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:39:47.669147629Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:39:48.101561701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:39:48.107198782Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T06:39:48.487128708Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T06:39:48.521434358Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:39:48.522818692Z caller=checkpoint.go:378 msg="writing series" size="703 kB"
level=info ts=2026-02-16T06:39:48.768520412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="49 kB" avg_comp="16 kB" total_uncomp="322 kB" avg_uncomp="107 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:39:48.771956452Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T06:39:49.539495435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="282 kB" avg_uncomp="282 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:39:49.542948842Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=info ts=2026-02-16T06:39:52.21626327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:39:52.223955871Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:39:52.223986705Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:39:52.224021964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:39:52.230545497Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:39:52.272647234Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-02-16T06:39:53.521761589Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:39:55.098684099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:39:55.101976268Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:39:55.442879711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:39:55.445925284Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T06:39:56.023636801Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T06:39:56.023660881Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=7
level=warn ts=2026-02-16T06:39:58.5218953Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:39:59.77287805Z caller=checkpoint.go:378 msg="writing series" size="509 kB"
ts=2026-02-16T06:40:01.282486986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:50438"
level=info ts=2026-02-16T06:40:02.3477607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:40:02.351455112Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:40:02.988317634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="323 B" avg_comp="323 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:40:02.991974089Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:40:03.521823525Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:40:03.521850498Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:40:03.521858743Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:40:03.521868405Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-02-16T06:40:03.521879152Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:40:04.169260596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:40:04.175346173Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T06:40:05.542331137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:40:05.549085812Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T06:40:05.861111783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:40:07.27251083Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-02-16T06:40:08.520907977Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:40:10.820406763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:40:10.824066336Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T06:40:11.022693244Z caller=checkpoint.go:378 msg="writing series" size="245 kB"
level=info ts=2026-02-16T06:40:11.557638986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:40:11.560996942Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:40:13.047241736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:40:13.050680403Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T06:40:13.521414501Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:40:14.595081495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:40:14.598485155Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T06:40:14.772009408Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=debug ts=2026-02-16T06:40:15.456980407Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:40:16.707863186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="344 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:40:16.711291096Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T06:40:17.102901576Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T06:40:18.520867236Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:40:18.522012959Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=info ts=2026-02-16T06:40:18.760628903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:40:18.764027663Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:40:20.15147083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:40:20.154929943Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:40:22.272754913Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=warn ts=2026-02-16T06:40:23.521695609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:40:25.480997644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:40:25.484786403Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T06:40:26.022456793Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=info ts=2026-02-16T06:40:28.372323026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:40:28.376223956Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:40:28.376243962Z caller=flush.go:240 component=ingester loop=8 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:40:28.37626568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:40:28.379315709Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=warn ts=2026-02-16T06:40:28.521483301Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:40:29.774009557Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T06:40:29.774033981Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=9
level=info ts=2026-02-16T06:40:29.944292822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="282 kB" avg_uncomp="282 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:40:29.948266757Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=info ts=2026-02-16T06:40:30.095618368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="426 kB" avg_comp="142 kB" total_uncomp="4.3 MB" avg_uncomp="1.4 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:40:30.099262492Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:40:30.39713079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:40:30.402070628Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:40:31.386623861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="301 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:40:31.389779961Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:40:32.818876767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:40:32.823006455Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T06:40:32.910405856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:40:32.913625784Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:40:33.356750616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:40:33.360236957Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:40:33.36026208Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:40:33.360287511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:40:33.363169482Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T06:40:33.502758115Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:40:33.503801006Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T06:40:33.521017323Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:40:33.521053405Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:40:33.521081424Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:40:33.521091859Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:40:33.522210848Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=info ts=2026-02-16T06:40:34.502019385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="928 B" avg_comp="928 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:40:34.506854213Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T06:40:35.862657484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
ts=2026-02-16T06:40:36.268553624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:38278"
level=debug ts=2026-02-16T06:40:37.271976383Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
level=info ts=2026-02-16T06:40:37.527785604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="49 kB" avg_comp="16 kB" total_uncomp="322 kB" avg_uncomp="107 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:40:37.531503399Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
ts=2026-02-16T06:40:38.172598529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:41244"
level=info ts=2026-02-16T06:40:38.189880013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:40:38.193174529Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=info ts=2026-02-16T06:40:38.508700723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:40:38.51561379Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T06:40:38.521706903Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:40:41.022497605Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=info ts=2026-02-16T06:40:41.165866432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:40:41.169173188Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:40:42.246823827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:40:42.250100757Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T06:40:42.338600111Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T06:40:43.521354305Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:40:44.459878691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:40:44.46403168Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=debug ts=2026-02-16T06:40:44.772065174Z caller=checkpoint.go:378 msg="writing series" size="30 kB"
level=info ts=2026-02-16T06:40:45.320698766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=5 total_comp="16 kB" avg_comp="3.2 kB" total_uncomp="143 kB" avg_uncomp="29 kB" full=5 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:40:45.324035665Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:40:46.400983872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:40:46.404491396Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=info ts=2026-02-16T06:40:47.333077029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:40:47.336622714Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T06:40:48.521405161Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:40:48.522516831Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=debug ts=2026-02-16T06:40:52.272492665Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=info ts=2026-02-16T06:40:52.554408576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:40:52.559173845Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T06:40:52.68041117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:40:52.683928212Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T06:40:53.521071959Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:40:55.642291192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:40:55.645465838Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T06:40:56.02309113Z caller=checkpoint.go:378 msg="writing series" size="792 kB"
level=debug ts=2026-02-16T06:40:56.023114637Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-02-16T06:40:58.521387859Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:40:59.772716953Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=info ts=2026-02-16T06:40:59.783835767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:40:59.78711684Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T06:41:01.287040275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:33588"
level=info ts=2026-02-16T06:41:01.651132995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:41:01.654610773Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T06:41:02.462774554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:41:02.46620448Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:41:02.640458849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T06:41:03.52177649Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:41:03.52180464Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:41:03.521814589Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:41:03.521827825Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.8 kB" series=1
level=warn ts=2026-02-16T06:41:03.521852885Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:41:03.527315153Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-02-16T06:41:03.530796063Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.008999293s
level=info ts=2026-02-16T06:41:05.577357209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:41:05.582027977Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
ts=2026-02-16T06:41:05.864435875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=warn ts=2026-02-16T06:41:08.521478045Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:41:09.2679791Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:41:11.168902101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.8 kB" avg_comp="443 B" total_uncomp="1.4 kB" avg_uncomp="344 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:41:11.172641156Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:41:12.883340741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="426 kB" avg_comp="142 kB" total_uncomp="4.3 MB" avg_uncomp="1.4 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:41:12.887135682Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T06:41:13.521757444Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:41:13.661107401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:41:13.667167585Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:41:13.667187158Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:41:13.667209536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:41:13.672304561Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:41:14.06818606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:41:14.071629261Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:41:16.339766304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:41:16.343121918Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T06:41:18.521912Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:41:19.40985448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:41:19.413264072Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:41:19.591731285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:41:19.595033491Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:41:20.865297213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:41:20.86882568Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T06:41:22.005395307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:41:22.009064283Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=warn ts=2026-02-16T06:41:23.520936035Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:41:24.580962675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="301 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:41:24.584235897Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T06:41:24.584256386Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:41:24.584278866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="301 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:41:24.586950189Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:41:27.25004272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:41:27.256510257Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:41:27.688614897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="282 kB" avg_uncomp="282 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:41:27.692661988Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=info ts=2026-02-16T06:41:27.780013717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:41:27.783087065Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:41:27.896147164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:41:27.89939784Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:41:27.931526162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:41:27.934357571Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=warn ts=2026-02-16T06:41:28.521614858Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:41:28.611934551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:41:28.61507786Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:41:30.095708322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:41:30.099756435Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T06:41:30.7404047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:41:30.745481118Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-02-16T06:41:31.289272663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:47884"
level=debug ts=2026-02-16T06:41:33.503286793Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:41:33.504350403Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:41:33.520961974Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T06:41:33.520998008Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:41:33.521016057Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:41:33.521024247Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:41:33.521046143Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:41:33.521107292Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=debug ts=2026-02-16T06:41:33.52130855Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=info ts=2026-02-16T06:41:33.714942415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=5 total_comp="16 kB" avg_comp="3.2 kB" total_uncomp="143 kB" avg_uncomp="29 kB" full=5 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:41:33.7183843Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T06:41:34.684294717Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T06:41:35.865720531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=info ts=2026-02-16T06:41:36.268435907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="49 kB" avg_comp="16 kB" total_uncomp="322 kB" avg_uncomp="107 kB" full=3 labels="{stream=\"knative-eventing\"}"
ts=2026-02-16T06:41:36.272570692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:47724"
level=error ts=2026-02-16T06:41:36.273747118Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:41:36.273765042Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:41:36.273787869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="49 kB" avg_comp="16 kB" total_uncomp="322 kB" avg_uncomp="107 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:41:36.276763537Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T06:41:36.991838643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:41:36.995165726Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T06:41:37.271808926Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
ts=2026-02-16T06:41:38.176506507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:36936"
level=warn ts=2026-02-16T06:41:38.521753507Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:41:39.641519079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:41:39.645933028Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T06:41:40.109326868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="36 kB" avg_comp="18 kB" total_uncomp="301 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:41:40.112509522Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:41:41.010660376Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T06:41:41.02178828Z caller=checkpoint.go:378 msg="writing series" size="131 kB"
level=warn ts=2026-02-16T06:41:43.521704184Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:41:44.773344817Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-02-16T06:41:44.773384263Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=4
level=info ts=2026-02-16T06:41:45.953067337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:41:45.95650252Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:41:47.308791546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T06:41:47.312319867Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c65062bfa%3A19c65062bfb%3A81d98baf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T06:41:47.31234136Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:41:47.312378832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T06:41:47.316031935Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c6506244d%3A19c65277166%3A4f5959e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T06:41:48.521400893Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:41:48.521486951Z caller=checkpoint.go:378 msg="writing series" size="164 kB"
level=debug ts=2026-02-16T06:41:52.272191648Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T06:41:52.669354549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:41:52.674878936Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T06:41:53.521013266Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:41:54.975503185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:41:54.978957374Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:41:55.77147316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="49 kB" avg_comp="16 kB" total_uncomp="322 kB" avg_uncomp="107 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:41:55.775110616Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T06:41:56.022192533Z caller=checkpoint.go:378 msg="writing series" size="371 kB"
level=info ts=2026-02-16T06:41:56.78787684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:41:56.79116647Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T06:41:58.520878001Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:41:59.033813299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="426 kB" avg_comp="142 kB" total_uncomp="4.3 MB" avg_uncomp="1.4 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:41:59.037874401Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:41:59.705083574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T06:41:59.709163799Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c6506244d%3A19c65277166%3A4f5959e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T06:41:59.772302829Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T06:41:59.871591274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:41:59.875019113Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T06:42:01.291432002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:46214"
level=info ts=2026-02-16T06:42:01.366045741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:42:01.369519123Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:42:01.369533794Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:42:01.369552995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:42:01.372411451Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:42:03.521147634Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:42:03.521170423Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:42:03.521178314Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:42:03.521172512Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:42:03.522492483Z caller=checkpoint.go:378 msg="writing series" size="158 kB"
ts=2026-02-16T06:42:05.867481594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=info ts=2026-02-16T06:42:06.253156127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:42:06.256656372Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:42:06.474993029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T06:42:06.475062156Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-02-16T06:42:06.480438384Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c65058cb8%3A19c652e884a%3A4a5dbc42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T06:42:07.271545866Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=info ts=2026-02-16T06:42:07.573566284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="36 kB" avg_comp="18 kB" total_uncomp="301 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:42:07.576914891Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:42:07.744411047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.8 kB" avg_comp="447 B" total_uncomp="1.4 kB" avg_uncomp="344 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:42:07.748129716Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:42:08.325625337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:42:08.329395196Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=warn ts=2026-02-16T06:42:08.521370592Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:42:08.913531904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:42:08.917058917Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:42:09.085013221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:42:09.088349631Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:42:09.088384994Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:42:09.088413444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:42:09.091265208Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:42:10.338849862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:42:10.34622025Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:42:10.526730446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:42:10.529994512Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:42:11.02281448Z caller=checkpoint.go:378 msg="writing series" size="554 kB"
level=debug ts=2026-02-16T06:42:11.02283832Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=7
level=info ts=2026-02-16T06:42:11.65690665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:42:11.66019273Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T06:42:13.521679625Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:42:13.738335179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:42:13.741765806Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:42:14.226270744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:42:14.22977655Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:42:14.229795155Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:42:14.22981618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:42:14.232563404Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T06:42:14.772230864Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=info ts=2026-02-16T06:42:16.164788237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:42:16.169394754Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T06:42:17.021413934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T06:42:17.026800377Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c65058cb8%3A19c652e884a%3A4a5dbc42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T06:42:17.277355156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="282 kB" avg_uncomp="282 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:42:17.281105779Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=error ts=2026-02-16T06:42:17.281125294Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:42:17.281148727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="282 kB" avg_uncomp="282 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:42:17.284251221Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=info ts=2026-02-16T06:42:18.149762043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:42:18.152883246Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T06:42:18.521888123Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-02-16T06:42:18.521889487Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:42:20.152827525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:42:20.157275821Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:42:20.157295261Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:42:20.157317813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:42:20.161088993Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T06:42:22.202744759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:42:22.206499923Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=debug ts=2026-02-16T06:42:22.2715881Z caller=checkpoint.go:378 msg="writing series" size="4.4 kB"
level=warn ts=2026-02-16T06:42:23.521137513Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:42:25.060647606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:42:25.064159715Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:42:25.943518949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T06:42:25.946918828Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c6506244d%3A19c65277166%3A4f5959e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T06:42:26.022308585Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=info ts=2026-02-16T06:42:26.964171878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="49 kB" avg_comp="16 kB" total_uncomp="322 kB" avg_uncomp="107 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:42:26.96740213Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T06:42:28.04007624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=5 total_comp="16 kB" avg_comp="3.2 kB" total_uncomp="143 kB" avg_uncomp="29 kB" full=5 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:42:28.043417341Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T06:42:28.270518942Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T06:42:28.521798165Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:42:29.772140729Z caller=checkpoint.go:378 msg="writing series" size="4.5 kB"
level=info ts=2026-02-16T06:42:30.75294123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:42:30.756103539Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:42:30.756124515Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:42:30.75614688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:42:30.759072072Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:42:31.249169253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:42:31.252399061Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:42:32.188531688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:42:32.193003434Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T06:42:32.918272984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:42:32.92133621Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:42:32.921356387Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:42:32.921396437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:42:32.92400833Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T06:42:33.503249218Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:42:33.504321194Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:42:33.52084857Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:42:33.520869233Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:42:33.520878082Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:42:33.520898088Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:42:33.522041125Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=info ts=2026-02-16T06:42:33.964099211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="282 kB" avg_uncomp="282 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:42:33.96789386Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
ts=2026-02-16T06:42:35.868584805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:42:37.271590031Z caller=checkpoint.go:378 msg="writing series" size="470 kB"
level=warn ts=2026-02-16T06:42:38.521672127Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:42:38.81334181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:42:38.818631927Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:42:38.818648627Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:42:38.818666891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:42:38.821649704Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T06:42:38.851843145Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T06:42:41.021945026Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T06:42:41.823407338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="426 kB" avg_comp="142 kB" total_uncomp="4.3 MB" avg_uncomp="1.4 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:42:41.827630119Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:42:41.827645785Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:42:41.827665905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="426 kB" avg_comp="142 kB" total_uncomp="4.3 MB" avg_uncomp="1.4 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:42:41.831181504Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:42:43.384953477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="713 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:42:43.388304248Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:42:43.493644816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T06:42:43.49919083Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c65058cb8%3A19c652e884a%3A4a5dbc42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T06:42:43.521911256Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:42:44.772110487Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
level=info ts=2026-02-16T06:42:47.305184188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:42:47.308891973Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:42:47.514611102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:42:47.520013277Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T06:42:48.521655642Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:42:48.521681012Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T06:42:49.690328803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:42:49.693755246Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:42:49.693774941Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:42:49.693795878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:42:49.696612405Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:42:49.723801779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:42:49.726639476Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:42:50.775318928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="36 kB" avg_comp="18 kB" total_uncomp="301 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:42:50.778571483Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:42:52.272175404Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=info ts=2026-02-16T06:42:52.513975285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:42:52.517389496Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:42:52.591480672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:42:52.594432276Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T06:42:52.727786679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:42:52.731262315Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=warn ts=2026-02-16T06:42:53.52124296Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:42:53.646540258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.8 kB" avg_comp="447 B" total_uncomp="1.4 kB" avg_uncomp="344 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:42:53.649987142Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:42:54.536538121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:42:54.539917723Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T06:42:56.021999104Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=debug ts=2026-02-16T06:42:56.022020555Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=12
level=info ts=2026-02-16T06:42:56.505353852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:42:56.509967932Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:42:56.50998775Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:42:56.510010486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:42:56.51386342Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T06:42:57.942951576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:42:57.946616098Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:42:58.423894789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="282 kB" avg_uncomp="282 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:42:58.427440711Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=warn ts=2026-02-16T06:42:58.521326029Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:42:59.368253264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="426 kB" avg_comp="142 kB" total_uncomp="4.3 MB" avg_uncomp="1.4 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:42:59.372235397Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:42:59.663013558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:42:59.666164515Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:42:59.66618455Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:42:59.666207006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:42:59.668860074Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T06:42:59.772412075Z caller=checkpoint.go:378 msg="writing series" size="253 kB"
level=info ts=2026-02-16T06:43:00.339053183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:43:00.343586176Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T06:43:03.521832846Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:43:03.521860518Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:43:03.521868528Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:43:03.521915777Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:43:03.523194183Z caller=checkpoint.go:378 msg="writing series" size="2.5 MB"
level=debug ts=2026-02-16T06:43:03.523206227Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.8 MB" series=2
level=info ts=2026-02-16T06:43:04.249084833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:43:04.252592353Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:43:05.031791079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:43:05.035040394Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T06:43:05.86994634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=info ts=2026-02-16T06:43:06.676000515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:43:06.679626511Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:43:07.053656421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:43:07.056960208Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:43:07.271548376Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=info ts=2026-02-16T06:43:07.372843637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:43:07.376312581Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:43:07.916978951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:43:07.923298276Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T06:43:08.521461126Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:43:09.920894513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:43:09.92702938Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T06:43:10.646992238Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T06:43:11.021961006Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T06:43:11.195409143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="49 kB" avg_comp="12 kB" total_uncomp="322 kB" avg_uncomp="80 kB" idle=1 full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:43:11.198818754Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T06:43:13.521259905Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:43:14.771581883Z caller=checkpoint.go:378 msg="writing series" size="208 kB"
level=info ts=2026-02-16T06:43:16.451496655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:43:16.455299951Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T06:43:18.520870312Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:43:18.522049171Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=debug ts=2026-02-16T06:43:18.713859926Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:43:18.715616812Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T06:43:19.181896019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:43:19.1852692Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:43:19.922406981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:43:19.925957273Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=info ts=2026-02-16T06:43:20.311003176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:43:20.314499596Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:43:22.271815696Z caller=checkpoint.go:378 msg="writing series" size="427 kB"
level=warn ts=2026-02-16T06:43:23.521834523Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:43:23.620083477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:43:23.623609771Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:43:24.170457083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=5 total_comp="16 kB" avg_comp="3.2 kB" total_uncomp="143 kB" avg_uncomp="29 kB" full=5 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:43:24.173808019Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:43:24.173826239Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:43:24.173848782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=5 total_comp="16 kB" avg_comp="3.2 kB" total_uncomp="143 kB" avg_uncomp="29 kB" full=5 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:43:24.176699672Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:43:25.520033746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T06:43:25.524045265Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c6506244d%3A19c65277166%3A4f5959e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T06:43:26.032834329Z caller=checkpoint.go:378 msg="writing series" size="9.2 MB"
level=debug ts=2026-02-16T06:43:26.032858966Z caller=checkpoint.go:390 msg="flushing series" totalSize="10 MB" series=6
level=info ts=2026-02-16T06:43:26.374265126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="571 B" avg_comp="571 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:43:26.37768671Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:43:27.265915006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="426 kB" avg_comp="142 kB" total_uncomp="4.3 MB" avg_uncomp="1.4 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:43:27.26977044Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T06:43:28.521732675Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:43:29.772524398Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=info ts=2026-02-16T06:43:31.638318874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:43:31.64208237Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:43:32.763657583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:43:32.767151967Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=info ts=2026-02-16T06:43:33.01187435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:43:33.016621324Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T06:43:33.502914756Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:43:33.503965189Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:43:33.521075955Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:43:33.521094676Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:43:33.521103227Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:43:33.521150938Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:43:33.522211184Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T06:43:34.266127246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=5 total_comp="16 kB" avg_comp="3.2 kB" total_uncomp="143 kB" avg_uncomp="29 kB" full=5 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:43:34.269387149Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-cluster-samples-operator\"}"
ts=2026-02-16T06:43:35.871739588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:43:37.271483541Z caller=checkpoint.go:378 msg="writing series" size="939 B"
level=warn ts=2026-02-16T06:43:38.521678455Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:43:39.897968302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:43:39.901656944Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T06:43:40.177337478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T06:43:40.184277132Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c65058cb8%3A19c652e884a%3A4a5dbc42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T06:43:40.311761566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.8 kB" avg_comp="447 B" total_uncomp="1.4 kB" avg_uncomp="344 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:43:40.315156256Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T06:43:41.022080199Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=info ts=2026-02-16T06:43:42.636114692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="282 kB" avg_uncomp="282 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:43:42.639571609Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=info ts=2026-02-16T06:43:42.686627339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:43:42.690699228Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T06:43:43.521695065Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:43:43.816616877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:43:43.820692794Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T06:43:44.77235163Z caller=checkpoint.go:378 msg="writing series" size="242 kB"
level=debug ts=2026-02-16T06:43:45.429264493Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:43:45.528632808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:43:45.532288383Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T06:43:48.521232422Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:43:48.543754593Z caller=checkpoint.go:378 msg="writing series" size="31 MB"
level=debug ts=2026-02-16T06:43:48.543778255Z caller=checkpoint.go:390 msg="flushing series" totalSize="31 MB" series=6
level=info ts=2026-02-16T06:43:49.028233656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:43:49.03281661Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T06:43:49.581555152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:43:49.585094998Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:43:49.856831556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:43:49.860432936Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:43:49.860451372Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:43:49.860472752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:43:49.863351133Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:43:50.106131351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="36 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="104 kB" idle=1 full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:43:50.109596371Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:43:50.164682699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:43:50.168082704Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:43:50.168100784Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:43:50.16812372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:43:50.171006863Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:43:50.460814944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:43:50.467700039Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:43:51.077416595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:43:51.080919945Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=debug ts=2026-02-16T06:43:52.271414025Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=warn ts=2026-02-16T06:43:53.521469429Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:43:56.022180061Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-02-16T06:43:58.521317223Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:43:59.453674485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:43:59.457356237Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:43:59.666980729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:43:59.670194512Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T06:43:59.781742783Z caller=checkpoint.go:378 msg="writing series" size="8.5 MB"
level=debug ts=2026-02-16T06:43:59.781765883Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.5 MB" series=3
level=info ts=2026-02-16T06:44:03.521743279Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T06:44:03.521751989Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:44:03.521765814Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:44:03.521774041Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:44:03.521798279Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
ts=2026-02-16T06:44:05.873061901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=info ts=2026-02-16T06:44:06.438197231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:44:06.441518742Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:44:06.732377928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="49 kB" avg_comp="12 kB" total_uncomp="322 kB" avg_uncomp="80 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:44:06.735531631Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T06:44:07.271938315Z caller=checkpoint.go:378 msg="writing series" size="330 kB"
level=debug ts=2026-02-16T06:44:07.366931794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T06:44:07.787206689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:44:07.791679884Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T06:44:08.186150316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:55078"
level=warn ts=2026-02-16T06:44:08.521828854Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:44:08.877966287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=5 total_comp="16 kB" avg_comp="3.2 kB" total_uncomp="143 kB" avg_uncomp="29 kB" full=5 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:44:08.881061727Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T06:44:11.022262194Z caller=checkpoint.go:378 msg="writing series" size="900 B"
level=info ts=2026-02-16T06:44:11.455587846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="426 kB" avg_comp="142 kB" total_uncomp="4.3 MB" avg_uncomp="1.4 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:44:11.466608138Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T06:44:13.521215819Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:44:14.474070051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:44:14.477834397Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=info ts=2026-02-16T06:44:14.585377945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:44:14.58868616Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T06:44:14.772432736Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=info ts=2026-02-16T06:44:15.973837234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:44:15.976958355Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=info ts=2026-02-16T06:44:16.781238515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:44:16.784569586Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:44:17.374409076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T06:44:17.377927688Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c6506244d%3A19c65277166%3A4f5959e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T06:44:18.521207972Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:44:18.522317962Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T06:44:19.48055917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="323 B" avg_comp="323 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:44:19.483947276Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:44:19.591454112Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:44:20.874786805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:44:20.878591479Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:44:21.209099797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T06:44:21.213868258Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c65058cb8%3A19c652e884a%3A4a5dbc42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T06:44:22.271532318Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-02-16T06:44:23.521741981Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:44:26.02184665Z caller=checkpoint.go:378 msg="writing series" size="249 kB"
level=info ts=2026-02-16T06:44:26.678174761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:44:26.682515448Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T06:44:26.956538775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:44:26.959727886Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:44:28.007568541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:44:28.010549507Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T06:44:28.521547419Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:44:29.772111715Z caller=checkpoint.go:378 msg="writing series" size="282 kB"
level=debug ts=2026-02-16T06:44:29.772134718Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
ts=2026-02-16T06:44:31.30049054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:40432"
level=info ts=2026-02-16T06:44:31.567179037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:44:31.573606221Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:44:31.682861082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:44:31.688481009Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:44:31.757392054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:44:31.760466986Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:44:31.980185407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:44:31.983863751Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:44:32.062038524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.8 kB" avg_comp="447 B" total_uncomp="1.4 kB" avg_uncomp="344 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:44:32.065074625Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:44:32.065090047Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:44:32.065112341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.8 kB" avg_comp="447 B" total_uncomp="1.4 kB" avg_uncomp="344 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:44:32.067853914Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:44:33.196409206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:44:33.200203863Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:44:33.200226144Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:44:33.200247215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:44:33.20352881Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:44:33.238624811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="36 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="104 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:44:33.242225843Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:44:33.503058857Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:44:33.50410088Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:44:33.520861118Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:44:33.520879387Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:44:33.520887253Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:44:33.520891567Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:44:33.522010695Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
ts=2026-02-16T06:44:35.874343233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=info ts=2026-02-16T06:44:36.182809837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:44:36.186156432Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T06:44:37.272197444Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=warn ts=2026-02-16T06:44:38.521497749Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:44:39.148089995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:44:39.151536741Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:44:40.686681681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=2 total_comp="32 kB" avg_comp="16 kB" total_uncomp="291 kB" avg_uncomp="146 kB" idle=1 full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:44:40.689901282Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-service\"}"
level=debug ts=2026-02-16T06:44:41.021990369Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T06:44:42.310081426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:44:42.315924002Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T06:44:43.362800078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.8 kB" avg_comp="447 B" total_uncomp="1.4 kB" avg_uncomp="344 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:44:43.366457276Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T06:44:43.521624177Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:44:44.7717188Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=debug ts=2026-02-16T06:44:46.817716521Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:44:47.528674225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:44:47.531857179Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=debug ts=2026-02-16T06:44:48.09779713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T06:44:48.521808377Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:44:48.522712193Z caller=checkpoint.go:378 msg="writing series" size="908 kB"
level=info ts=2026-02-16T06:44:50.407251904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:44:50.41044653Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:44:52.271971367Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-02-16T06:44:53.38139546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="49 kB" avg_comp="12 kB" total_uncomp="322 kB" avg_uncomp="80 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:44:53.384890705Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T06:44:53.521064547Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:44:56.023851962Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T06:44:56.023875715Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=7
level=info ts=2026-02-16T06:44:56.309956222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="426 kB" avg_comp="142 kB" total_uncomp="4.3 MB" avg_uncomp="1.4 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:44:56.313842264Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:44:56.386741472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:44:56.390153581Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T06:44:58.52118569Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:44:59.351242295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:44:59.354882618Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T06:44:59.772259682Z caller=checkpoint.go:378 msg="writing series" size="521 kB"
level=info ts=2026-02-16T06:45:02.229951097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:45:02.234063165Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=warn ts=2026-02-16T06:45:03.520987818Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:45:03.521013601Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:45:03.521040768Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:45:03.521051913Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:45:03.522123446Z caller=checkpoint.go:378 msg="writing series" size="1.4 kB"
level=info ts=2026-02-16T06:45:05.070026536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T06:45:05.075972397Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c65058cb8%3A19c652e884a%3A4a5dbc42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-02-16T06:45:05.875695498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=debug ts=2026-02-16T06:45:07.272325134Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T06:45:07.405867357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=5 total_comp="16 kB" avg_comp="3.2 kB" total_uncomp="143 kB" avg_uncomp="29 kB" full=5 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:45:07.409808228Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-cluster-samples-operator\"}"
ts=2026-02-16T06:45:08.191198557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:52010"
level=info ts=2026-02-16T06:45:08.448168837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:45:08.451510026Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:45:08.451530122Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:45:08.45155143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:45:08.454349003Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=warn ts=2026-02-16T06:45:08.521209419Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:45:09.802070242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.8 kB" avg_comp="447 B" total_uncomp="1.4 kB" avg_uncomp="344 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:45:09.805405828Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T06:45:11.021972445Z caller=checkpoint.go:378 msg="writing series" size="245 kB"
level=info ts=2026-02-16T06:45:11.988699766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T06:45:11.992717341Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c6506244d%3A19c65277166%3A4f5959e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T06:45:13.52097214Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:45:14.771525767Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=info ts=2026-02-16T06:45:15.056258211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="36 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="104 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:45:15.06111843Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:45:15.331046492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:45:15.335318112Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T06:45:15.384388962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:45:15.387437973Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:45:16.519773769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:45:16.522875368Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:45:16.857758023Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:45:17.643147544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:45:17.646151712Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:45:17.741549793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:45:17.746123404Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T06:45:18.521241178Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:45:18.521344119Z caller=checkpoint.go:378 msg="writing series" size="71 kB"
level=info ts=2026-02-16T06:45:21.163337115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:45:21.166515636Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:45:22.00995976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:45:22.01673117Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:45:22.272292704Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=info ts=2026-02-16T06:45:23.115543619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:45:23.118614959Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:45:23.241928015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:45:23.245028619Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T06:45:23.521388769Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:45:25.817724533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:45:25.821513076Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T06:45:26.021608027Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=info ts=2026-02-16T06:45:26.30502731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:45:26.308108503Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:45:26.937083435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:45:26.940040325Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:45:27.144622701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:45:27.147699722Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=info ts=2026-02-16T06:45:28.268644241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="928 B" avg_comp="928 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:45:28.271788543Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T06:45:28.459935973Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T06:45:28.520888262Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:45:29.773491311Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T06:45:29.773521166Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.1 MB" series=9
level=debug ts=2026-02-16T06:45:33.503214408Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:45:33.504271788Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T06:45:33.50449013Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-02-16T06:45:33.504526714Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1968026 n=1
level=debug ts=2026-02-16T06:45:33.504568026Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1968026 n=1
level=info ts=2026-02-16T06:45:33.521730341Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:45:33.521745343Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:45:33.521752671Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:45:33.521755524Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:45:33.521822515Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=info ts=2026-02-16T06:45:34.448742331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:45:34.45193778Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T06:45:35.878055255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:45:36.2899156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:60278"
level=info ts=2026-02-16T06:45:37.076690387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=2 total_comp="32 kB" avg_comp="16 kB" total_uncomp="291 kB" avg_uncomp="146 kB" full=2 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:45:37.079893297Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-service\"}"
level=debug ts=2026-02-16T06:45:37.271559382Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=warn ts=2026-02-16T06:45:38.521244973Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:45:39.74526123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="49 kB" avg_comp="12 kB" total_uncomp="322 kB" avg_uncomp="80 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:45:39.748815188Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T06:45:41.022161696Z caller=checkpoint.go:378 msg="writing series" size="107 kB"
level=info ts=2026-02-16T06:45:42.364413137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:45:42.368157517Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:45:42.368177107Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:45:42.368199201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:45:42.371112682Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T06:45:43.521618331Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:45:44.772009915Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=info ts=2026-02-16T06:45:45.801893849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:45:45.807142787Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T06:45:48.521737066Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:45:48.521737836Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=info ts=2026-02-16T06:45:50.642561469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="426 kB" avg_comp="142 kB" total_uncomp="4.3 MB" avg_uncomp="1.4 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:45:50.646808588Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T06:45:51.329907884Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T06:45:52.271682761Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=warn ts=2026-02-16T06:45:53.521700994Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:45:54.711865233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:45:54.715878609Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:45:55.085614043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=5 total_comp="16 kB" avg_comp="3.2 kB" total_uncomp="143 kB" avg_uncomp="29 kB" full=5 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:45:55.088970568Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T06:45:56.022885655Z caller=checkpoint.go:378 msg="writing series" size="843 kB"
level=debug ts=2026-02-16T06:45:56.022909317Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-02-16T06:45:56.860155562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:45:56.864460061Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=warn ts=2026-02-16T06:45:58.521791964Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:45:58.807336635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:45:58.810771004Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:45:58.887935016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:45:58.89345395Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:45:58.931257273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T06:45:58.936296284Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c65058cb8%3A19c652e884a%3A4a5dbc42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T06:45:59.771422588Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
ts=2026-02-16T06:46:01.306739307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:47266"
level=info ts=2026-02-16T06:46:03.521282293Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:46:03.521313474Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T06:46:03.521311214Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:46:03.52131571Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.8 kB" series=1
level=info ts=2026-02-16T06:46:03.521325256Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T06:46:03.526519346Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-02-16T06:46:03.528757774Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.0075269s
level=info ts=2026-02-16T06:46:04.133408662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:46:04.138253352Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T06:46:04.179331078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:46:04.18262558Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:46:04.685022791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:46:04.688324962Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:46:05.505663508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:46:05.508990886Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=info ts=2026-02-16T06:46:05.591087477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:46:05.594517761Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:46:05.657603797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="379 B" total_uncomp="1.7 kB" avg_uncomp="343 B" idle=1 full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:46:05.660654929Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
ts=2026-02-16T06:46:05.879831067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=info ts=2026-02-16T06:46:05.958053866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T06:46:05.961825038Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c6506244d%3A19c65277166%3A4f5959e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T06:46:08.521527539Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:46:10.011980921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="36 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="104 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:46:10.016028198Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:46:10.815173717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:46:10.818483488Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T06:46:12.420814518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:46:12.42434081Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T06:46:13.521779804Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:46:13.609122148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:46:13.616048963Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:46:13.616065043Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:46:13.616091543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:46:13.622544434Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:46:14.140056261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:46:14.143488366Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:46:14.696992624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:46:14.700300514Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:46:17.800700587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=2 total_comp="32 kB" avg_comp="16 kB" total_uncomp="291 kB" avg_uncomp="146 kB" full=2 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:46:17.805636037Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-service\"}"
level=warn ts=2026-02-16T06:46:18.521307803Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:46:18.915861849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:46:18.918956598Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:46:21.23260528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:46:21.23595998Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:46:21.250012781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:46:21.253466252Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T06:46:22.03788147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:46:22.041030759Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T06:46:23.25145806Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T06:46:23.521103575Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:46:23.858724035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="49 kB" avg_comp="12 kB" total_uncomp="322 kB" avg_uncomp="80 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:46:23.861879436Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T06:46:25.446168183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T06:46:25.449224846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:46:25.452066577Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:46:27.855797046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:46:27.862714103Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T06:46:28.520879372Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:46:29.113613179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:46:29.117104192Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T06:46:33.503012876Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:46:33.504051812Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:46:33.521660116Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:46:33.521683856Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:46:33.521676845Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T06:46:33.521696981Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:46:33.521715529Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:46:33.521820832Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=debug ts=2026-02-16T06:46:33.522017926Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=info ts=2026-02-16T06:46:34.971060962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:46:34.974831431Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T06:46:35.881866174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:46:36.292761367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:39796"
level=debug ts=2026-02-16T06:46:37.272526146Z caller=checkpoint.go:378 msg="writing series" size="33 kB"
level=info ts=2026-02-16T06:46:38.026756668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:46:38.030391052Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:46:38.030410419Z caller=flush.go:240 component=ingester loop=8 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:46:38.030430837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:46:38.033472176Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=warn ts=2026-02-16T06:46:38.521485231Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:46:41.023057393Z caller=checkpoint.go:378 msg="writing series" size="131 kB"
level=info ts=2026-02-16T06:46:41.620804604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T06:46:41.626062513Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c65058cb8%3A19c652e884a%3A4a5dbc42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T06:46:43.521657116Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:46:44.069479773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="426 kB" avg_comp="142 kB" total_uncomp="4.3 MB" avg_uncomp="1.4 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:46:44.07362157Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T06:46:44.774777169Z caller=checkpoint.go:378 msg="writing series" size="1.7 MB"
level=debug ts=2026-02-16T06:46:44.774803313Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=4
level=info ts=2026-02-16T06:46:45.737358858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=5 total_comp="18 kB" avg_comp="3.5 kB" total_uncomp="143 kB" avg_uncomp="29 kB" full=5 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:46:45.7408918Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:46:46.679352566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T06:46:46.683837075Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c6506244d%3A19c65277166%3A4f5959e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T06:46:48.521818609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:46:48.52295754Z caller=checkpoint.go:378 msg="writing series" size="164 kB"
level=info ts=2026-02-16T06:46:49.200069808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:46:49.206497884Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:46:50.147037229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:46:50.150781629Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:46:51.021959321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:46:51.025349518Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:46:51.436320718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:46:51.439987118Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=info ts=2026-02-16T06:46:51.967481919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="383 B" total_uncomp="1.7 kB" avg_uncomp="343 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:46:51.970915573Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T06:46:52.272674895Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=debug ts=2026-02-16T06:46:52.634010448Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:46:52.874352944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:46:52.877650975Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:46:52.877670563Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:46:52.877699595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:46:52.880456679Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T06:46:53.52125568Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:46:54.76354109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:46:54.766837788Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=info ts=2026-02-16T06:46:55.301338096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:46:55.306025531Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T06:46:55.895420619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:46:55.898633148Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T06:46:56.023122377Z caller=checkpoint.go:378 msg="writing series" size="276 kB"
level=info ts=2026-02-16T06:46:56.043155502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:46:56.046137276Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:46:57.84144114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=2 total_comp="32 kB" avg_comp="16 kB" total_uncomp="291 kB" avg_uncomp="146 kB" full=2 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:46:57.84475952Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-service\"}"
level=warn ts=2026-02-16T06:46:58.521033428Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:46:59.772345155Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=info ts=2026-02-16T06:47:01.311837098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="341 B" avg_comp="341 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:47:01.315327853Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:47:03.52114748Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:47:03.521173274Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:47:03.521181977Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:47:03.521180057Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:47:03.522382528Z caller=checkpoint.go:378 msg="writing series" size="158 kB"
level=info ts=2026-02-16T06:47:04.576077235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:47:04.583430272Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:47:04.688637405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:47:04.691813253Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-02-16T06:47:05.884279005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=debug ts=2026-02-16T06:47:07.273007429Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-02-16T06:47:08.521396081Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:47:09.080943591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:47:09.084388348Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:47:09.474409279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="36 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="104 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:47:09.477987897Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:47:10.48366181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:47:10.487293546Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=debug ts=2026-02-16T06:47:11.023171064Z caller=checkpoint.go:378 msg="writing series" size="560 kB"
level=debug ts=2026-02-16T06:47:11.023196403Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-02-16T06:47:13.520873422Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:47:14.300150866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:47:14.303962741Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:47:14.772391001Z caller=checkpoint.go:378 msg="writing series" size="109 kB"
level=info ts=2026-02-16T06:47:16.428657077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:47:16.43380137Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T06:47:17.877487663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="49 kB" avg_comp="12 kB" total_uncomp="322 kB" avg_uncomp="80 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:47:17.881018947Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T06:47:18.346534498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="946 B" avg_comp="946 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:47:18.349883428Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T06:47:18.521459579Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:47:18.52257404Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T06:47:19.969634328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:47:19.974765571Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:47:20.040992267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:47:20.044320949Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T06:47:22.27216488Z caller=checkpoint.go:378 msg="writing series" size="4.4 kB"
level=warn ts=2026-02-16T06:47:23.521137438Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:47:24.734209762Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T06:47:25.475695534Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:47:25.477209823Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T06:47:26.022133529Z caller=checkpoint.go:378 msg="writing series" size="162 kB"
level=warn ts=2026-02-16T06:47:28.521779953Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:47:29.772509897Z caller=checkpoint.go:378 msg="writing series" size="4.5 kB"
level=info ts=2026-02-16T06:47:30.101513265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="426 kB" avg_comp="142 kB" total_uncomp="4.3 MB" avg_uncomp="1.4 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:47:30.105589385Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:47:30.241772913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:47:30.245039723Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:47:30.789511256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T06:47:30.793597213Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c6506244d%3A19c65277166%3A4f5959e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T06:47:30.86875304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=5 total_comp="18 kB" avg_comp="3.5 kB" total_uncomp="143 kB" avg_uncomp="29 kB" full=5 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:47:30.872202656Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:47:31.027719921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:47:31.031472377Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
ts=2026-02-16T06:47:31.312533223Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:36400"
level=debug ts=2026-02-16T06:47:33.503474843Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:47:33.504512515Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:47:33.521154054Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:47:33.521179116Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:47:33.521190686Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:47:33.521186778Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:47:33.522409506Z caller=checkpoint.go:378 msg="writing series" size="164 kB"
level=info ts=2026-02-16T06:47:35.337542869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:47:35.341409905Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
ts=2026-02-16T06:47:35.886274404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=info ts=2026-02-16T06:47:36.969280283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:47:36.972609806Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T06:47:37.272519372Z caller=checkpoint.go:378 msg="writing series" size="470 kB"
level=info ts=2026-02-16T06:47:38.052486708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T06:47:38.057811727Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c65058cb8%3A19c652e884a%3A4a5dbc42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T06:47:38.33070343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="383 B" total_uncomp="1.7 kB" avg_uncomp="343 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:47:38.334520582Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T06:47:38.521486854Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:47:39.107901756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:47:39.111208038Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T06:47:41.022503694Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=warn ts=2026-02-16T06:47:43.520964318Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:47:44.772304847Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
level=info ts=2026-02-16T06:47:47.135286275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:47:47.140457779Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:47:47.140475326Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:47:47.140496956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:47:47.145276899Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:47:47.333903179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="534 B" avg_comp="267 B" total_uncomp="545 B" avg_uncomp="272 B" idle=1 full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:47:47.337344291Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T06:47:48.521571947Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:47:48.522680947Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T06:47:49.148040839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:47:49.151679533Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T06:47:49.411411549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:47:49.418758867Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:47:49.864907626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:47:49.868798369Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:47:50.728218427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:47:50.73543312Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:47:51.65078325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=2 total_comp="32 kB" avg_comp="16 kB" total_uncomp="291 kB" avg_uncomp="146 kB" full=2 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:47:51.654396567Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-service\"}"
level=debug ts=2026-02-16T06:47:52.273008535Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=info ts=2026-02-16T06:47:52.498625699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:47:52.503887698Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T06:47:53.521422434Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:47:54.64590064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="36 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="104 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:47:54.6490666Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T06:47:54.649085591Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:47:54.649109318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="36 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="104 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:47:54.651681414Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:47:56.022330917Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=debug ts=2026-02-16T06:47:56.022352733Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=12
level=info ts=2026-02-16T06:47:58.349006721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="713 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:47:58.352532861Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T06:47:58.521704927Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:47:59.772474655Z caller=checkpoint.go:378 msg="writing series" size="307 kB"
level=debug ts=2026-02-16T06:47:59.981917523Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:48:01.250634338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:48:01.256186059Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:48:03.520816752Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:48:03.520842567Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:48:03.520851049Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:48:03.520934378Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:48:03.525421242Z caller=checkpoint.go:378 msg="writing series" size="2.3 MB"
level=debug ts=2026-02-16T06:48:03.525438393Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.6 MB" series=2
ts=2026-02-16T06:48:05.887683384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
level=info ts=2026-02-16T06:48:06.345069556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:48:06.348652168Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:48:07.232941856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:48:07.237531573Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T06:48:07.272745265Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=info ts=2026-02-16T06:48:08.207092013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:48:08.210889954Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=warn ts=2026-02-16T06:48:08.521554152Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:48:08.960401085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="49 kB" avg_comp="12 kB" total_uncomp="322 kB" avg_uncomp="80 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:48:08.963668493Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:48:08.963687003Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:48:08.963707717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="49 kB" avg_comp="12 kB" total_uncomp="322 kB" avg_uncomp="80 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:48:08.966303278Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T06:48:11.022180322Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T06:48:11.532923291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:48:11.536338018Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:48:13.286146107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="36 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="104 kB" full=3 labels="{stream=\"image-controller\"}"
level=info ts=2026-02-16T06:48:13.288841025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="426 kB" avg_comp="142 kB" total_uncomp="4.3 MB" avg_uncomp="1.4 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:48:13.28949246Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T06:48:13.292456716Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T06:48:13.520941333Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:48:13.549108148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:48:13.552240508Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:48:14.772593332Z caller=checkpoint.go:378 msg="writing series" size="271 kB"
level=debug ts=2026-02-16T06:48:14.798261754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T06:48:14.940650071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:48:14.944043891Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=info ts=2026-02-16T06:48:16.698592225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="534 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" idle=1 full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:48:16.70207717Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:48:17.90624313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:48:17.909503452Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T06:48:18.520942579Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:48:18.523124351Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=info ts=2026-02-16T06:48:20.744076895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="383 B" total_uncomp="1.7 kB" avg_uncomp="343 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:48:20.749139475Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T06:48:22.272296879Z caller=checkpoint.go:378 msg="writing series" size="436 kB"
level=debug ts=2026-02-16T06:48:22.272321298Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-02-16T06:48:22.449862211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:48:22.453297062Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T06:48:23.521219306Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:48:25.156244884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="49 kB" avg_comp="12 kB" total_uncomp="322 kB" avg_uncomp="80 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:48:25.159713752Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T06:48:25.47845324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T06:48:25.48339644Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c65058cb8%3A19c652e884a%3A4a5dbc42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T06:48:25.483416116Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:48:25.483438286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T06:48:25.488023937Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c65058cb8%3A19c652e884a%3A4a5dbc42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T06:48:26.027744946Z caller=checkpoint.go:378 msg="writing series" size="9.2 MB"
level=debug ts=2026-02-16T06:48:26.02776704Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.2 MB" series=1
level=debug ts=2026-02-16T06:48:26.138195631Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:48:26.366701454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T06:48:26.371346633Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c6506244d%3A19c65277166%3A4f5959e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-02-16T06:48:26.371387193Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:48:26.371409351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T06:48:26.374693827Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c6506244d%3A19c65277166%3A4f5959e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T06:48:26.659649498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:48:26.662816668Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=warn ts=2026-02-16T06:48:28.521402808Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:48:28.616735572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=5 total_comp="18 kB" avg_comp="3.5 kB" total_uncomp="143 kB" avg_uncomp="29 kB" full=5 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:48:28.620047147Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:48:29.73683961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="571 B" avg_comp="571 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:48:29.740258105Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T06:48:29.772334981Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=info ts=2026-02-16T06:48:31.179262316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="534 B" avg_comp="267 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:48:31.182516061Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:48:33.502978344Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:48:33.504043304Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:48:33.520798558Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:48:33.520820144Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:48:33.520828074Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:48:33.520861537Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:48:33.522915956Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T06:48:35.41517605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:48:35.418772231Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:48:35.52616426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:48:35.532529778Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T06:48:35.888959572Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-v2482-71315b6b 10.129.0.59:7946"
level=debug ts=2026-02-16T06:48:37.272528417Z caller=checkpoint.go:378 msg="writing series" size="939 B"
level=warn ts=2026-02-16T06:48:38.521796021Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:48:39.034340266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:48:39.039767759Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:48:40.122791902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:48:40.126457345Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:48:40.126476953Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:48:40.126500996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:48:40.129116687Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:48:40.837994083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T06:48:40.841708814Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c6506244d%3A19c65277166%3A4f5959e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T06:48:41.022354923Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=info ts=2026-02-16T06:48:42.674150747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:48:42.677422463Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T06:48:43.521610714Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:48:43.893616605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T06:48:43.898999157Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c65058cb8%3A19c652e884a%3A4a5dbc42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T06:48:44.772326392Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
level=info ts=2026-02-16T06:48:47.151761329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=2 total_comp="32 kB" avg_comp="16 kB" total_uncomp="291 kB" avg_uncomp="146 kB" full=2 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:48:47.155508611Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-service\"}"
level=error ts=2026-02-16T06:48:47.155527803Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:48:47.155553966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=2 total_comp="32 kB" avg_comp="16 kB" total_uncomp="291 kB" avg_uncomp="146 kB" full=2 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:48:47.158379054Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-service\"}"
level=info ts=2026-02-16T06:48:47.492184243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:48:47.495498051Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:48:48.423726094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="49 kB" avg_comp="12 kB" total_uncomp="322 kB" avg_uncomp="80 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:48:48.427080254Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T06:48:48.521418706Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:48:48.547269743Z caller=checkpoint.go:378 msg="writing series" size="31 MB"
level=debug ts=2026-02-16T06:48:48.5472896Z caller=checkpoint.go:390 msg="flushing series" totalSize="31 MB" series=6
level=info ts=2026-02-16T06:48:48.798544959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:48:48.8021381Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:48:49.107593507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:48:49.112022325Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:48:49.112039322Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:48:49.112060911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:48:49.115998915Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T06:48:51.780407889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="36 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="104 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:48:51.784620128Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:48:52.272952566Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=warn ts=2026-02-16T06:48:53.5218249Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:48:54.620553295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:48:54.623662683Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:48:54.623683697Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:48:54.623705831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:48:54.626178297Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T06:48:54.834495913Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T06:48:55.440075256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:48:55.444197434Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T06:48:55.509622071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="426 kB" avg_comp="142 kB" total_uncomp="4.3 MB" avg_uncomp="1.4 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:48:55.512996508Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:48:55.513014805Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:48:55.51303643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="426 kB" avg_comp="142 kB" total_uncomp="4.3 MB" avg_uncomp="1.4 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:48:55.516068065Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T06:48:56.022417578Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-02-16T06:48:58.52150953Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:48:59.544527316Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T06:48:59.78213826Z caller=checkpoint.go:378 msg="writing series" size="8.7 MB"
level=debug ts=2026-02-16T06:48:59.782162334Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.8 MB" series=3
level=info ts=2026-02-16T06:48:59.918480861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:48:59.922056834Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:49:01.765675475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:49:01.768880181Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:49:02.61611399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:49:02.619255745Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:49:03.091491461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=2 total_comp="32 kB" avg_comp="16 kB" total_uncomp="291 kB" avg_uncomp="146 kB" full=2 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:49:03.094814953Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-service\"}"
level=info ts=2026-02-16T06:49:03.521594366Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T06:49:03.521595284Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:49:03.521618785Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:49:03.521627003Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:49:03.522808284Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T06:49:04.675540459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="534 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:49:04.678563626Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:49:04.678583993Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:49:04.678606473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="534 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:49:04.68131202Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:49:05.082523316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:49:05.085680954Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
ts=2026-02-16T06:49:05.89016423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:49:06.302341931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:58032"
level=info ts=2026-02-16T06:49:06.53557546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:49:06.538439846Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:49:06.538456397Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:49:06.538477041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:49:06.540934376Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T06:49:07.272959743Z caller=checkpoint.go:378 msg="writing series" size="400 kB"
ts=2026-02-16T06:49:08.206007028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:60938"
level=info ts=2026-02-16T06:49:08.271575127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:49:08.275732555Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T06:49:08.521104447Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:49:11.022152512Z caller=checkpoint.go:378 msg="writing series" size="902 B"
level=info ts=2026-02-16T06:49:11.22875842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="516 B" avg_comp="258 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:49:11.232344966Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:49:11.232380603Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:49:11.232401163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="534 B" avg_comp="267 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:49:11.235235543Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:49:11.543098357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="383 B" total_uncomp="1.7 kB" avg_uncomp="343 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:49:11.546388966Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:49:11.734236129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T06:49:11.739322821Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c65058cb8%3A19c652e884a%3A4a5dbc42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T06:49:11.880827718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="426 kB" avg_comp="142 kB" total_uncomp="4.3 MB" avg_uncomp="1.4 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:49:11.884850132Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:49:12.605107162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:49:12.608564904Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:49:12.825563363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T06:49:12.829543977Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c6506244d%3A19c65277166%3A4f5959e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T06:49:13.521198955Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:49:14.772375831Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=info ts=2026-02-16T06:49:15.664760881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="534 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:49:15.66814379Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T06:49:15.956977789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:49:15.960434838Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T06:49:17.204403942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=6 total_comp="18 kB" avg_comp="2.9 kB" total_uncomp="158 kB" avg_uncomp="26 kB" idle=1 full=5 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:49:17.20794673Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:49:17.840606123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:49:17.843932921Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:49:17.843951475Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:49:17.843972193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:49:17.848145896Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T06:49:18.521273519Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:49:18.522419372Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T06:49:18.654647802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:49:18.661143741Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:49:20.05571568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:49:20.05916102Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=debug ts=2026-02-16T06:49:22.27226107Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T06:49:23.315086334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:49:23.322932027Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T06:49:23.521310699Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:49:24.994532598Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:49:25.212092576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="534 B" avg_comp="267 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:49:25.215453858Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:49:25.556386831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:49:25.560464319Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6506243c%3A19c65064d99%3A7c930579?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T06:49:25.56048723Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:49:25.560510474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="182 B" avg_comp="182 B" total_uncomp="23 kB" avg_uncomp="23 kB" idle=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T06:49:25.563403378Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c6506243c%3A19c65298539%3A4f87e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-02-16T06:49:26.022862943Z caller=checkpoint.go:378 msg="writing series" size="300 kB"
level=info ts=2026-02-16T06:49:26.379726617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:49:26.383100371Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:49:26.392136806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:49:26.395040566Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:49:26.395053664Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:49:26.39507252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:49:26.39772155Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T06:49:28.521452851Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:49:29.772529808Z caller=checkpoint.go:378 msg="writing series" size="386 kB"
level=debug ts=2026-02-16T06:49:29.772553801Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=8
level=debug ts=2026-02-16T06:49:33.502778808Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:49:33.503842782Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T06:49:33.503866796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:49:33.508262815Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T06:49:33.521471455Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:49:33.521487488Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:49:33.52149486Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:49:33.521496096Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:49:33.522616807Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=info ts=2026-02-16T06:49:35.343762733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:49:35.347297269Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
ts=2026-02-16T06:49:35.892200871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=info ts=2026-02-16T06:49:36.059593788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:49:36.063292214Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:49:36.780661964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="36 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="104 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:49:36.784082145Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T06:49:37.272423694Z caller=checkpoint.go:378 msg="writing series" size="51 kB"
ts=2026-02-16T06:49:38.207821411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:41736"
level=warn ts=2026-02-16T06:49:38.521747649Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:49:38.595983233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:49:38.599796105Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:49:39.040980827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:49:39.045119751Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T06:49:40.56219435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T06:49:40.565441491Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c6506243c%3A19c65298539%3A4f87e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T06:49:40.648643469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="49 kB" avg_comp="12 kB" total_uncomp="322 kB" avg_uncomp="80 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:49:40.651618664Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T06:49:41.022697034Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T06:49:42.825016603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=2 total_comp="32 kB" avg_comp="16 kB" total_uncomp="291 kB" avg_uncomp="146 kB" full=2 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:49:42.828168148Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-service\"}"
level=info ts=2026-02-16T06:49:43.058856629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:49:43.061989069Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T06:49:43.146287816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:49:43.149200216Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T06:49:43.521053895Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:49:44.772094406Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=warn ts=2026-02-16T06:49:48.521813681Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:49:48.523279873Z caller=checkpoint.go:378 msg="writing series" size="866 kB"
level=info ts=2026-02-16T06:49:48.692196273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:49:48.696721914Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:49:48.696742651Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:49:48.696764211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:49:48.700959396Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T06:49:48.954441994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T06:49:48.982485203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:49:48.985743817Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=info ts=2026-02-16T06:49:49.000938378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:49:49.003784208Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T06:49:49.248416478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:49:49.25203556Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:49:50.227642041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="426 kB" avg_comp="142 kB" total_uncomp="4.3 MB" avg_uncomp="1.4 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:49:50.231198874Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:49:51.259887587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="534 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:49:51.263050527Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T06:49:52.273047394Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-02-16T06:49:53.521391755Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:49:55.993226183Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T06:49:56.022955167Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T06:49:56.022972303Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=7
level=info ts=2026-02-16T06:49:57.77050365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:49:57.774750607Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T06:49:58.521235698Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:49:58.650574784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="383 B" total_uncomp="1.7 kB" avg_uncomp="343 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:49:58.654050585Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T06:49:59.41180292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:49:59.415750103Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T06:49:59.773067252Z caller=checkpoint.go:378 msg="writing series" size="525 kB"
level=info ts=2026-02-16T06:50:00.612195701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="534 B" avg_comp="267 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:50:00.616021581Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T06:50:01.28977913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T06:50:01.295168524Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c65058cb8%3A19c652e884a%3A4a5dbc42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-02-16T06:50:01.321478644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:51970"
level=info ts=2026-02-16T06:50:02.174012324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:50:02.177157872Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:50:02.177177979Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:50:02.177198876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:50:02.179845082Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:50:03.521457055Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:50:03.521482358Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T06:50:03.521482507Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:50:03.52149454Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:50:03.522592352Z caller=checkpoint.go:378 msg="writing series" size="1.4 kB"
level=info ts=2026-02-16T06:50:05.228853655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:50:05.232608896Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T06:50:05.745085913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T06:50:05.748694779Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c6506244d%3A19c65277166%3A4f5959e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-02-16T06:50:05.894475385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=debug ts=2026-02-16T06:50:07.273039328Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-02-16T06:50:08.521724925Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:50:09.345459705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=6 total_comp="18 kB" avg_comp="2.9 kB" total_uncomp="158 kB" avg_uncomp="26 kB" full=6 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:50:09.348848019Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:50:09.348868533Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:50:09.348890821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=6 total_comp="18 kB" avg_comp="2.9 kB" total_uncomp="158 kB" avg_uncomp="26 kB" full=6 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:50:09.351795817Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:50:10.89989062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:50:10.907128908Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T06:50:11.022489931Z caller=checkpoint.go:378 msg="writing series" size="245 kB"
level=warn ts=2026-02-16T06:50:13.520986764Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:50:14.772931041Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=info ts=2026-02-16T06:50:15.112966041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:50:15.116253898Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
level=info ts=2026-02-16T06:50:15.145401799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:50:15.150064416Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T06:50:17.809466054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:50:17.813036995Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T06:50:18.52099993Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:50:18.522141057Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
level=info ts=2026-02-16T06:50:18.529210436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:50:18.532672595Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:50:20.10420511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T06:50:20.107812648Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c6506243c%3A19c65298539%3A4f87e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T06:50:21.154654344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:50:21.159090157Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T06:50:22.272100743Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=warn ts=2026-02-16T06:50:23.521651009Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:50:23.718880378Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:50:24.387980509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:50:24.391149908Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:50:24.485403866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="36 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="104 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:50:24.48846219Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:50:25.327747349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:50:25.33083662Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:50:25.330854059Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:50:25.330874847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:50:25.333448512Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T06:50:26.02217142Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=debug ts=2026-02-16T06:50:26.176963986Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T06:50:27.950702357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=6 total_comp="18 kB" avg_comp="2.9 kB" total_uncomp="158 kB" avg_uncomp="26 kB" full=6 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:50:27.95577157Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T06:50:28.520997378Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:50:29.772972455Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T06:50:29.77299878Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=9
level=info ts=2026-02-16T06:50:30.004163288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:50:30.007745078Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-02-16T06:50:31.323422552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:35244"
level=info ts=2026-02-16T06:50:31.372835744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:50:31.376286391Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:50:31.437489036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:50:31.441586601Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T06:50:33.218155054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:50:33.222064613Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T06:50:33.502821851Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.58:11211 count=1
level=info ts=2026-02-16T06:50:33.503873458Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T06:50:33.520950722Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:50:33.520961266Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:50:33.520977388Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:50:33.52098883Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T06:50:33.522107074Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
ts=2026-02-16T06:50:35.896186579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-2sdsw-273d962c 10.128.0.57:7946"
ts=2026-02-16T06:50:36.307984836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:57982"
level=info ts=2026-02-16T06:50:36.770930649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="426 kB" avg_comp="142 kB" total_uncomp="4.3 MB" avg_uncomp="1.4 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:50:36.775968561Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T06:50:37.140982094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:50:37.144318575Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T06:50:37.272762351Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=warn ts=2026-02-16T06:50:38.521750543Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:50:39.440898562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="49 kB" avg_comp="12 kB" total_uncomp="322 kB" avg_uncomp="80 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:50:39.444454891Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T06:50:40.68776089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=2 total_comp="32 kB" avg_comp="16 kB" total_uncomp="291 kB" avg_uncomp="146 kB" full=2 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:50:40.691155082Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-service\"}"
level=debug ts=2026-02-16T06:50:41.022238947Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=info ts=2026-02-16T06:50:41.522589521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:50:41.526208337Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=info ts=2026-02-16T06:50:42.044572574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:50:42.048157765Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:50:42.953105578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:50:42.956664637Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T06:50:43.520873094Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:50:44.77282051Z caller=checkpoint.go:378 msg="writing series" size="34 kB"
level=info ts=2026-02-16T06:50:46.680864115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="525 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T06:50:46.684225779Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c65062957%3A19c65062958%3A2716f1e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T06:50:48.521214416Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:50:48.522337558Z caller=checkpoint.go:378 msg="writing series" size="3.4 kB"
level=info ts=2026-02-16T06:50:51.263665595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=6 total_comp="18 kB" avg_comp="2.9 kB" total_uncomp="158 kB" avg_uncomp="26 kB" full=6 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T06:50:51.267162867Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6505ac89%3A19c65062bfb%3A6ddb8acd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T06:50:51.843886633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="534 B" avg_comp="267 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T06:50:51.84694126Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6506244d%3A19c65062bfa%3Aa3ea37bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T06:50:52.272046269Z caller=checkpoint.go:378 msg="writing series" size="5.5 kB"
level=warn ts=2026-02-16T06:50:53.521875744Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:50:53.600117929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T06:50:53.604918175Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c65058cb8%3A19c652e884a%3A4a5dbc42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T06:50:53.786129315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T06:50:53.789606079Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c6506244d%3A19c65277166%3A4f5959e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T06:50:54.106132897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T06:50:54.109182393Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6506243c%3A19c65062bfa%3A8c5eb9ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T06:50:54.885216059Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T06:50:56.022739211Z caller=checkpoint.go:378 msg="writing series" size="894 kB"
level=debug ts=2026-02-16T06:50:56.022765319Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=7
level=info ts=2026-02-16T06:50:58.131407221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="2.2 kB" avg_comp="442 B" total_uncomp="1.7 kB" avg_uncomp="343 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:50:58.135627159Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:50:58.135646318Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:50:58.135669746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="2.2 kB" avg_comp="446 B" total_uncomp="1.7 kB" avg_uncomp="343 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:50:58.138539799Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T06:50:58.521207279Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:50:59.772056253Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=info ts=2026-02-16T06:50:59.990724664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T06:50:59.994416715Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c650f3e15%3A19c65114802%3A25cb00d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T06:51:00.185783175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T06:51:00.189264144Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6506244c%3A19c650a9142%3Aab9fb2b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T06:51:00.67255286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6420f76fdd8317fe immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="40 kB" avg_uncomp="40 kB" full=1 labels="{stream=\"konflux-lmcs-managed\"}"
level=error ts=2026-02-16T06:51:00.676200607Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6420f76fdd8317fe/19c65178728%3A19c651a4266%3A78dabad3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-lmcs-managed\"}"
ts=2026-02-16T06:51:01.325505674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:59628"
level=info ts=2026-02-16T06:51:02.215983263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="861 kB" avg_comp="861 kB" total_uncomp="5.7 MB" avg_uncomp="5.7 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T06:51:02.220933576Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c650593e5%3A19c65220e7e%3A567872ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T06:51:02.354265682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T06:51:02.361457864Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c65058da3%3A19c65078506%3Abccd419b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T06:51:03.521691469Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T06:51:03.521721053Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T06:51:03.521728928Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T06:51:03.521719389Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:51:03.522862639Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.8 kB" series=1
level=info ts=2026-02-16T06:51:03.526013438Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-02-16T06:51:03.530040175Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.008086671s
level=info ts=2026-02-16T06:51:05.629026166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="8.1 MB" avg_uncomp="8.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T06:51:05.634233095Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c65058850%3A19c65163d8a%3Aaaa467a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-02-16T06:51:05.898599288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a7e144a7 10.128.0.60:7946"
level=warn ts=2026-02-16T06:51:08.521025624Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T06:51:09.543267091Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T06:51:09.544930547Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T06:51:12.17049439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="2.2 kB" avg_comp="446 B" total_uncomp="1.7 kB" avg_uncomp="343 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T06:51:12.1739333Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c65092810%3A19c65092811%3A8d3fd778?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T06:51:13.521050038Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:51:15.210666219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="722 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T06:51:15.214154475Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6506244c%3A19c65062bfb%3Ac4db1924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T06:51:17.056051582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="589 B" avg_comp="589 B" total_uncomp="1.0 kB" avg_uncomp="1.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T06:51:17.059521335Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c65068de9%3A19c650735e2%3A78ea1ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T06:51:17.11185248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T06:51:17.114981645Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c6506243c%3A19c65298539%3A4f87e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-02-16T06:51:18.520923932Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:51:18.54604682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="788 kB" avg_comp="788 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T06:51:18.550635406Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c65062957%3A19c651de4aa%3A32744065?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T06:51:18.628829421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="36 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="104 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T06:51:18.631949043Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c650593e5%3A19c6506243d%3Acb30c47d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T06:51:20.23990375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:51:20.24366782Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:51:20.243688416Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T06:51:20.243712333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b0edc32e19c3e5a5 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="671 kB" avg_uncomp="671 kB" full=1 labels="{stream=\"integration1-fobm\"}"
level=error ts=2026-02-16T06:51:20.24666267Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b0edc32e19c3e5a5/19c650f2a8d%3A19c651bc39a%3A6d8b0c62?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-fobm\"}"
level=info ts=2026-02-16T06:51:20.893988468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="49 kB" avg_comp="12 kB" total_uncomp="322 kB" avg_uncomp="80 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T06:51:20.897455737Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c650593e7%3A19c651226ab%3Abcf2038d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T06:51:22.258097661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="358 kB" avg_uncomp="358 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T06:51:22.261414542Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6505a9f3%3A19c65089b71%3A6c6d3c57?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T06:51:23.521030026Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:51:24.020529256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T06:51:24.024040738Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6506243c%3A19c65062958%3Acc807f35?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T06:51:24.19042102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="426 kB" avg_comp="142 kB" total_uncomp="4.3 MB" avg_uncomp="1.4 MB" full=2 synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T06:51:24.194208829Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6505989e%3A19c6514c0d1%3A8e6b46fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T06:51:28.045435647Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T06:51:28.448577441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=2 total_comp="32 kB" avg_comp="16 kB" total_uncomp="291 kB" avg_uncomp="146 kB" full=2 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T06:51:28.453797941Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6506244c%3A19c651fef86%3A80c4587f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-service\"}"
level=warn ts=2026-02-16T06:51:28.521600754Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T06:51:28.809781339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T06:51:28.813157434Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6506243c%3A19c6507dff1%3Ae77189f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T06:51:29.758501067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3bbd1ac5c01daba8 immediate=false num_chunks=1 total_comp="222 kB" avg_comp="222 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-wupm\"}"
level=error ts=2026-02-16T06:51:29.762357318Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3bbd1ac5c01daba8/19c650f2e75%3A19c65174941%3Aa956a5a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-wupm\"}"
level=info ts=2026-02-16T06:51:29.785397185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T06:51:29.788290025Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c65062bfa%3A19c6506f592%3Ac8cefc0c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"